From 66a8f4e3fc552d2160d1a1206666f7f54fd9b0a2 Mon Sep 17 00:00:00 2001
From: Nicolas Pernoud <github@ninico.fr>
Date: Thu, 10 Sep 2020 15:47:28 +0200
Subject: [PATCH] feat: working version 3.5
---
docker-compose.yml | 6 +--
nominatim-3.1/start.sh | 45 -------------------
nominatim-3.1/startapache.sh | 3 --
nominatim-3.1/startpostgres.sh | 4 --
{nominatim-3.1 => nominatim-3.5}/Dockerfile | 38 ++++++++--------
{nominatim-3.1 => nominatim-3.5}/README.md | 31 ++++++++-----
{nominatim-3.1 => nominatim-3.5}/init.sh | 11 +++--
{nominatim-3.1 => nominatim-3.5}/local.php | 11 ++---
.../nominatim.conf | 1 -
nominatim-3.5/start.sh | 16 +++++++
nominatim-3.5/startapache.sh | 6 +++
nominatim-3.5/startpostgres.sh | 4 ++
photon/Dockerfile | 2 +-
13 files changed, 80 insertions(+), 98 deletions(-)
delete mode 100755 nominatim-3.1/start.sh
delete mode 100755 nominatim-3.1/startapache.sh
delete mode 100755 nominatim-3.1/startpostgres.sh
rename {nominatim-3.1 => nominatim-3.5}/Dockerfile (57%)
mode change 100755 => 100644
rename {nominatim-3.1 => nominatim-3.5}/README.md (58%)
mode change 100755 => 100644
rename {nominatim-3.1 => nominatim-3.5}/init.sh (61%)
mode change 100755 => 100644
rename {nominatim-3.1 => nominatim-3.5}/local.php (55%)
mode change 100755 => 100644
rename {nominatim-3.1 => nominatim-3.5}/nominatim.conf (95%)
mode change 100755 => 100644
create mode 100755 nominatim-3.5/start.sh
create mode 100755 nominatim-3.5/startapache.sh
create mode 100755 nominatim-3.5/startpostgres.sh
diff --git a/docker-compose.yml b/docker-compose.yml
index f54ad5f..244b8a8 100755
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -3,12 +3,12 @@ services:
nominatim:
# image: nominatim
build:
- context: ./nominatim-3.1
+ context: ./nominatim-3.5
dockerfile: Dockerfile
restart: unless-stopped
volumes:
- - ./nominatimdata/postgresdata:/var/lib/postgresql/9.5/main
- - ./nominatim-3.1/local.php:/app/src/build/settings/local.php
+ #- ./nominatimdata/postgresdata:/var/lib/postgresql/12/main
+ - ./nominatim-3.5/local.php:/app/src/build/settings/local.php
ports:
- 6432:5432
- 7070:8080
diff --git a/nominatim-3.1/start.sh b/nominatim-3.1/start.sh
deleted file mode 100755
index 7698317..0000000
--- a/nominatim-3.1/start.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/bash
-service postgresql start
-/usr/sbin/apache2ctl start
-
-## Set up nominatim updates ###
-
-while true; do
-
- result=`sudo -u nominatim ./src/build/utils/update.php --check-for-updates`
-
- if [ "$result" = "Database up to date." ]; then
- sleepy=300
- echo "The Nominatim database in already up to date. Sleeping for ${sleepy}s..."
- sleep $sleepy
- else
-
- curl http://photon:2322/api
- isPhotonAlive=$?
-
- if [ $isPhotonAlive -ne 0 ]; then
- sleepy=300
- echo "Photon is not available. Sleeping for ${sleepy}s, in order to let Photon wake up..."
- sleep $sleepy
- continue
- fi
-
- sudo -u nominatim ./src/build/utils/update.php --init-updates
- sudo -u nominatim ./src/build/utils/update.php --import-osmosis --no-index
- curl -v http://photon:2322/nominatim-update # this line triggers the Photon update following the Nominatim update
-
- sleepy=1800
- echo "Sleeping for ${sleepy}s, in order to let Photon update..."
- sleep $sleepy
-
- sudo -u nominatim ./src/build/utils/update.php --index
-
- fi
-
-done
-
-
-
-## Follow log ###
-
-#tail -f /var/log/postgresql/postgresql-9.5-main.log
diff --git a/nominatim-3.1/startapache.sh b/nominatim-3.1/startapache.sh
deleted file mode 100755
index a204587..0000000
--- a/nominatim-3.1/startapache.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-/usr/sbin/apache2ctl -D FOREGROUND
-tail -f /var/log/apache2/error.log
\ No newline at end of file
diff --git a/nominatim-3.1/startpostgres.sh b/nominatim-3.1/startpostgres.sh
deleted file mode 100755
index 320fefe..0000000
--- a/nominatim-3.1/startpostgres.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-service postgresql start
-tail -f /var/log/postgresql/postgresql-9.5-main.log
\ No newline at end of file
diff --git a/nominatim-3.1/Dockerfile b/nominatim-3.5/Dockerfile
old mode 100755
new mode 100644
similarity index 57%
rename from nominatim-3.1/Dockerfile
rename to nominatim-3.5/Dockerfile
index faa71cd..525f231
--- a/nominatim-3.1/Dockerfile
+++ b/nominatim-3.5/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:xenial
+FROM ubuntu:focal
ENV DEBIAN_FRONTEND noninteractive
ENV LANG C.UTF-8
@@ -7,14 +7,15 @@ RUN apt-get -y update -qq && \
apt-get -y install locales && \
locale-gen en_US.UTF-8 && \
update-locale LANG=en_US.UTF-8 && \
- apt-get install -y build-essential cmake g++ libboost-dev libboost-system-dev \
- libboost-filesystem-dev libexpat1-dev zlib1g-dev libxml2-dev\
- libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev \
- postgresql-server-dev-9.5 postgresql-9.5-postgis-2.2 postgresql-contrib-9.5 \
- apache2 php php-pgsql libapache2-mod-php php-pear php-db \
- php-intl git curl sudo \
- python-pip libboost-python-dev \
- osmosis && \
+ apt-get install -o APT::Install-Recommends="false" -o APT::Install-Suggests="false" -y \
+ build-essential cmake g++ libboost-dev libboost-system-dev \
+ libboost-filesystem-dev libexpat1-dev zlib1g-dev \
+ libbz2-dev libpq-dev libproj-dev \
+ postgresql-server-dev-12 postgresql-12-postgis-3 \
+ postgresql-contrib postgresql-12-postgis-3-scripts \
+ apache2 php php-pgsql libapache2-mod-php \
+ php-intl python3-setuptools python3-dev python3-pip \
+ python3-psycopg2 python3-tidylib git curl sudo && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
rm -rf /tmp/* /var/tmp/*
@@ -22,17 +23,17 @@ RUN apt-get -y update -qq && \
WORKDIR /app
# Configure postgres
-RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/9.5/main/pg_hba.conf && \
- echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf
+RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/12/main/pg_hba.conf && \
+ echo "listen_addresses='*'" >> /etc/postgresql/12/main/postgresql.conf
+
+# Osmium install to run continuous updates
+RUN pip3 install osmium
# Nominatim install
-ENV NOMINATIM_VERSION v3.1.0
+ENV NOMINATIM_VERSION v3.5.1
RUN git clone --recursive https://github.com/openstreetmap/Nominatim ./src
RUN cd ./src && git checkout tags/$NOMINATIM_VERSION && git submodule update --recursive --init && \
- mkdir build && cd build && cmake .. && make
-
-# Osmium install to run continuous updates
-RUN pip install osmium
+ mkdir build && cd build && cmake .. && make -j`nproc`
# Apache configure
COPY local.php /app/src/build/settings/local.php
@@ -40,10 +41,7 @@ COPY nominatim.conf /etc/apache2/sites-enabled/000-default.conf
# Load initial data
RUN curl http://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz
-
-# Create nominatim user
-RUN useradd -m -p password1234 nominatim && \
- chown -R nominatim:nominatim ./src
+RUN chmod o=rwx /app/src/build
EXPOSE 5432
EXPOSE 8080
diff --git a/nominatim-3.1/README.md b/nominatim-3.5/README.md
old mode 100755
new mode 100644
similarity index 58%
rename from nominatim-3.1/README.md
rename to nominatim-3.5/README.md
index 8e58bf9..cc2803d
--- a/nominatim-3.1/README.md
+++ b/nominatim-3.5/README.md
@@ -1,24 +1,24 @@
-# Nominatim Docker (Nominatim version 3.1)
+# Nominatim Docker (Nominatim version 3.5)
1. Build
```
- docker build -t nominatim .
+ docker build --pull --rm -t nominatim .
```
2. Copy <your_country>.osm.pbf to a local directory (i.e. /home/me/nominatimdata)
3. Initialize Nominatim Database
```
- docker run -t -v /home/me/nominatimdata:/data nominatim sh /app/init.sh /data/merged.osm.pbf postgresdata 4
+ docker run -t -v /home/me/nominatimdata:/data nominatim sh /app/init.sh /data/<your_country>.osm.pbf postgresdata 4
```
Where 4 is the number of threads to use during import. In general the import of data in postgres is a very time consuming
process that may take hours or days. If you run this process on a multiprocessor system make sure that it makes the best use
- of it. You can delete the /home/me/nominatimdata/merged.osm.pbf once the import is finished.
+ of it. You can delete the /home/me/nominatimdata/<your_country>.osm.pbf once the import is finished.
4. After the import is finished the /home/me/nominatimdata/postgresdata folder will contain the full postgress binaries of
a postgis/nominatim database. The easiest way to start the nominatim as a single node is the following:
```
- docker run --restart=always -p 6432:5432 -p 7070:8080 -d -v /home/me/nominatimdata/postgresdata:/var/lib/postgresql/9.5/main nominatim sh /app/start.sh
+ docker run --restart=always -p 6432:5432 -p 7070:8080 -d --name nominatim -v /home/me/nominatimdata/postgresdata:/var/lib/postgresql/12/main nominatim bash /app/start.sh
```
5. Advanced configuration. If necessary you can split the osm installation into a database and restservice layer
@@ -26,7 +26,7 @@
In order to set the nominatib-db only node:
```
- docker run --restart=always -p 6432:5432 -d -v /home/me/nominatimdata/postgresdata:/var/lib/postgresql/9.5/main nominatim sh /app/startpostgres.sh
+ docker run --restart=always -p 6432:5432 -d -v /home/me/nominatimdata/postgresdata:/var/lib/postgresql/12/main nominatim sh /app/startpostgres.sh
```
After doing this create the /home/me/nominatimdata/conf folder and copy there the docker/local.php file. Then uncomment the following line:
@@ -41,7 +41,7 @@
```
6. Configure incremental update. By default CONST_Replication_Url configured for Monaco.
-If you want a different update source, you will need to declare `CONST_Replication_Url` in local.php. Documentation [here] (https://github.com/openstreetmap/Nominatim/blob/master/docs/Import-and-Update.md#updates). For example, to use the daily country extracts diffs for Gemany from geofabrik add the following:
+If you want a different update source, you will need to declare `CONST_Replication_Url` in local.php. Documentation [here] (https://github.com/openstreetmap/Nominatim/blob/master/docs/admin/Import-and-Update.md#updates). For example, to use the daily country extracts diffs for Gemany from geofabrik add the following:
```
@define('CONST_Replication_Url', 'http://download.geofabrik.de/europe/germany-updates');
```
@@ -52,15 +52,26 @@ If you want a different update source, you will need to declare `CONST_Replicati
# Update
-Full documentation for Nominatim update available [here](https://github.com/openstreetmap/Nominatim/blob/master/docs/Import-and-Update.md#updates). For a list of other methods see the output of:
+Full documentation for Nominatim update available [here](https://github.com/openstreetmap/Nominatim/blob/master/docs/admin/Import-and-Update.md#updates). For a list of other methods see the output of:
```
- docker exec -it nominatim sudo -u nominatim ./src/build/utils/update.php --help
+ docker exec -it nominatim sudo -u postgres ./src/build/utils/update.php --help
+ ```
+
+To initialise the updates run
+ ```
+ docker exec -it nominatim sudo -u postgres ./src/build/utils/update.php --init-updates
```
The following command will keep your database constantly up to date:
```
- docker exec -it nominatim sudo -u nominatim ./src/build/utils/update.php --import-osmosis-all
+ docker exec -it nominatim sudo -u postgres ./src/build/utils/update.php --import-osmosis-all
```
If you have imported multiple country extracts and want to keep them
up-to-date, have a look at the script in
[issue #60](https://github.com/openstreetmap/Nominatim/issues/60).
+
+# Docker image upgrade to 3.5
+
+With 3.5 we have switched to Ubuntu 20.04 (LTS) which uses PostgreSQL 12. If you want to reuse your old data dictionary without importing the data again you have to make sure to migrate the data from PostgreSQL 11 to 12 with a command like ```pg_upgrade``` (see: [https://www.postgresql.org/docs/current/pgupgrade.html](https://www.postgresql.org/docs/current/pgupgrade.html)).
+
+You can try a script like [https://github.com/tianon/docker-postgres-upgrade](https://github.com/tianon/docker-postgres-upgrade) with some modifications.
\ No newline at end of file
diff --git a/nominatim-3.1/init.sh b/nominatim-3.5/init.sh
old mode 100755
new mode 100644
similarity index 61%
rename from nominatim-3.1/init.sh
rename to nominatim-3.5/init.sh
index 032c421..1d59262
--- a/nominatim-3.1/init.sh
+++ b/nominatim-3.5/init.sh
@@ -2,17 +2,20 @@ OSMFILE=$1
PGDIR=$2
THREADS=$3
+rm -rf /data/$PGDIR && \
mkdir -p /data/$PGDIR && \
chown postgres:postgres /data/$PGDIR && \
export PGDATA=/data/$PGDIR && \
-sudo -u postgres /usr/lib/postgresql/9.5/bin/initdb -D /data/$PGDIR && \
-sudo -u postgres /usr/lib/postgresql/9.5/bin/pg_ctl -D /data/$PGDIR start && \
+sudo -u postgres /usr/lib/postgresql/12/bin/initdb -D /data/$PGDIR && \
+sudo -u postgres /usr/lib/postgresql/12/bin/pg_ctl -D /data/$PGDIR start && \
sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='nominatim'" | grep -q 1 || sudo -u postgres createuser -s nominatim && \
sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -u postgres createuser -SDR www-data && \
sudo -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" && \
+useradd -m -p password1234 nominatim && \
+chown -R nominatim:nominatim ./src && \
sudo -u nominatim ./src/build/utils/setup.php --osm-file $OSMFILE --all --threads $THREADS && \
-sudo -u postgres psql postgres -tAc "CREATE INDEX nodes_index ON public.planet_osm_ways USING gin (nodes);"
-sudo -u postgres /usr/lib/postgresql/9.5/bin/pg_ctl -D /data/$PGDIR stop && \
+sudo -u nominatim ./src/build/utils/check_import_finished.php && \
+sudo -u postgres /usr/lib/postgresql/12/bin/pg_ctl -D /data/$PGDIR stop && \
sudo chown -R postgres:postgres /data/$PGDIR
\ No newline at end of file
diff --git a/nominatim-3.1/local.php b/nominatim-3.5/local.php
old mode 100755
new mode 100644
similarity index 55%
rename from nominatim-3.1/local.php
rename to nominatim-3.5/local.php
index 0a22151..85a16f7
--- a/nominatim-3.1/local.php
+++ b/nominatim-3.5/local.php
@@ -1,15 +1,12 @@
<?php
// Paths
- @define('CONST_Postgresql_Version', '9.5');
- @define('CONST_Postgis_Version', '2.2');
+ @define('CONST_Postgresql_Version', '12');
+ @define('CONST_Postgis_Version', '3');
// Website settings
@define('CONST_Website_BaseURL', '/');
- @define('CONST_Replication_Url', 'https://download.geofabrik.de/europe/france-updates');
+ @define('CONST_Replication_Url', 'http://download.geofabrik.de/europe/monaco-updates');
@define('CONST_Replication_MaxInterval', '86400'); // Process each update separately, osmosis cannot merge multiple updates
@define('CONST_Replication_Update_Interval', '86400'); // How often upstream publishes diffs
@define('CONST_Replication_Recheck_Interval', '900'); // How long to sleep if no update found yet
@define('CONST_Pyosmium_Binary', '/usr/local/bin/pyosmium-get-changes');
-
- //@define('CONST_Database_DSN', 'pgsql://nominatim:password1234@192.168.1.128:6432/nominatim'); // <driver>://<username>:<password>@<host>:<port>/<database>
-
-?>
+ //@define('CONST_Database_DSN', 'pgsql:host=192.168.1.128;port=6432;user=nominatim;password=password1234;dbname=nominatim'); <driver>:host=<host>;port=<port>;user=<username>;password=<password>;dbname=<database>
diff --git a/nominatim-3.1/nominatim.conf b/nominatim-3.5/nominatim.conf
old mode 100755
new mode 100644
similarity index 95%
rename from nominatim-3.1/nominatim.conf
rename to nominatim-3.5/nominatim.conf
index c8df724..9031c6f
--- a/nominatim-3.1/nominatim.conf
+++ b/nominatim-3.5/nominatim.conf
@@ -1,5 +1,4 @@
Listen 8080
-ServerName localhost
<VirtualHost *:8080>
DocumentRoot /app/src/build/website
CustomLog /var/log/apache2/access.log combined
diff --git a/nominatim-3.5/start.sh b/nominatim-3.5/start.sh
new file mode 100755
index 0000000..5a8840b
--- /dev/null
+++ b/nominatim-3.5/start.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+stopServices() {
+ service apache2 stop
+ service postgresql stop
+}
+trap stopServices TERM
+
+#chown -Rf postgres:postgres /var/lib/postgresql/12/main
+
+service postgresql start
+service apache2 start
+
+# fork a process and wait for it
+tail -f /var/log/postgresql/postgresql-12-main.log &
+wait
diff --git a/nominatim-3.5/startapache.sh b/nominatim-3.5/startapache.sh
new file mode 100755
index 0000000..0f69c4d
--- /dev/null
+++ b/nominatim-3.5/startapache.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+cp /data/local.php /app/src/build/settings/local.php
+
+/usr/sbin/apache2ctl -D FOREGROUND
+tail -f /var/log/apache2/error.log
+
diff --git a/nominatim-3.5/startpostgres.sh b/nominatim-3.5/startpostgres.sh
new file mode 100755
index 0000000..b892b0d
--- /dev/null
+++ b/nominatim-3.5/startpostgres.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+service postgresql start
+tail -f /var/log/postgresql/postgresql-12-main.log
\ No newline at end of file
diff --git a/photon/Dockerfile b/photon/Dockerfile
index 610a443..dd03eb4 100755
--- a/photon/Dockerfile
+++ b/photon/Dockerfile
@@ -1,7 +1,7 @@
FROM openjdk:8-jre
WORKDIR /photon
-ADD https://github.com/komoot/photon/releases/download/0.3.0/photon-0.3.0.jar /photon/photon.jar
+ADD https://github.com/komoot/photon/releases/download/0.3.3/photon-0.3.3.jar /photon/photon.jar
COPY entrypoint.sh ./entrypoint.sh
EXPOSE 2322
--
GitLab