forked from Overv/openstreetmap-tile-server
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
4 changed files
with
303 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,207 @@ | ||
#!/bin/sh | ||
|
||
set -e | ||
|
||
#------------------------------------------------------------------------------ | ||
# AJT - change directory to mod_tile directory so that we can run replag | ||
# and other things directly from this script when run from cron. | ||
# Change the actual location to wherever installed locally. | ||
#------------------------------------------------------------------------------ | ||
export PATH=.:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin | ||
ACCOUNT=renderer | ||
cd /home/$ACCOUNT/src/mod_tile/ | ||
|
||
#------------------------------------------------------------------------------ | ||
# Extra OSM2PGSQL_OPTIONS may need setting if a tag transform script is | ||
# in use. See https://github.com/SomeoneElseOSM/SomeoneElse-style and | ||
# http://wiki.openstreetmap.org/wiki/User:SomeoneElse/Ubuntu_1404_tileserver_load | ||
# The database name always needs setting. | ||
#------------------------------------------------------------------------------ | ||
OSMOSIS_BIN=osmosis | ||
OSM2PGSQL_BIN=osm2pgsql | ||
TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py | ||
|
||
DBNAME=gis | ||
OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua -C 2048 --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style" | ||
|
||
#------------------------------------------------------------------------------ | ||
# When using trim_osc.py we can define either a bounding box (such as this | ||
# example for England and Wales) or a polygon. | ||
# See https://github.com/zverik/regional . | ||
# This area will usually correspond to the data originally loaded. | ||
#------------------------------------------------------------------------------ | ||
TRIM_POLY_FILE="/var/lib/mod_tile/data.poly" | ||
TRIM_OPTIONS="-d $DBNAME" | ||
#TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27" | ||
TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE" | ||
|
||
BASE_DIR=/var/lib/mod_tile | ||
LOG_DIR=/var/log/tiles/ | ||
WORKOSM_DIR=$BASE_DIR/.osmosis | ||
|
||
LOCK_FILE=/tmp/openstreetmap-update-expire-lock.txt | ||
CHANGE_FILE=$BASE_DIR/changes.osc.gz | ||
EXPIRY_FILE=$BASE_DIR/dirty_tiles | ||
STOP_FILE=$BASE_DIR/stop.txt | ||
|
||
OSMOSISLOG=$LOG_DIR/osmosis.log | ||
PGSQLLOG=$LOG_DIR/osm2pgsql.log | ||
EXPIRYLOG=$LOG_DIR/expiry.log | ||
RUNLOG=$LOG_DIR/run.log | ||
|
||
#------------------------------------------------------------------------------ | ||
# The tile expiry section below can re-render, delete or dirty expired tiles. | ||
# By default, tiles between EXPIRY_MINZOOM and EXPIRY_MAXZOOM are rerendered. | ||
# "render_expired" can optionally delete (and/or dirty) tiles above a certail | ||
# threshold rather than rendering them. | ||
# Here we expire (but don't immediately rerender) tiles between zoom levels | ||
# 13 and 18 and delete between 19 and 20. | ||
#------------------------------------------------------------------------------ | ||
EXPIRY_MINZOOM=13 | ||
EXPIRY_TOUCHFROM=13 | ||
EXPIRY_DELETEFROM=19 | ||
EXPIRY_MAXZOOM=20 | ||
|
||
#************************************************************************* | ||
#************************************************************************* | ||
|
||
m_info() | ||
{ | ||
echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG" | ||
} | ||
|
||
m_error() | ||
{ | ||
echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ [error] $1" >> "$RUNLOG" | ||
|
||
m_info "resetting state" | ||
/bin/cp $WORKOSM_DIR/last.state.txt $WORKOSM_DIR/state.txt || true | ||
|
||
rm "$CHANGE_FILE" || true | ||
rm "$EXPIRY_FILE.$$" || true | ||
rm "$LOCK_FILE" | ||
exit | ||
} | ||
|
||
m_ok() | ||
{ | ||
echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG" | ||
} | ||
|
||
getlock() | ||
{ | ||
if [ -s $1 ]; then | ||
if [ "$(ps -p `cat $1` | wc -l)" -gt 1 ]; then | ||
return 1 #false | ||
fi | ||
fi | ||
|
||
echo $$ >"$1" | ||
return 0 #true | ||
} | ||
|
||
freelock() | ||
{ | ||
rm "$1" | ||
rm "$CHANGE_FILE" | ||
} | ||
|
||
|
||
if [ $# -eq 1 ] ; then | ||
m_info "Initialising Osmosis replication system to $1" | ||
mkdir $WORKOSM_DIR | ||
$OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" | ||
wget "http://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt | ||
mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt | ||
sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt | ||
else | ||
# make sure the lockfile is removed when we exit and then claim it | ||
|
||
if ! getlock "$LOCK_FILE"; then | ||
m_info "pid `cat $LOCK_FILE` still running" | ||
exit 3 | ||
fi | ||
|
||
if [ -e $STOP_FILE ]; then | ||
m_info "stopped" | ||
exit 2 | ||
fi | ||
|
||
# ----------------------------------------------------------------------------- | ||
# Add disk space check from https://github.com/zverik/regional | ||
# ----------------------------------------------------------------------------- | ||
MIN_DISK_SPACE_MB=500 | ||
|
||
if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_bavail*st.f_frsize/1024/1024 > $MIN_DISK_SPACE_MB else 0)"`; then | ||
m_info "there is less than $MIN_DISK_SPACE_MB MB left" | ||
exit 4 | ||
fi | ||
|
||
seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2` | ||
|
||
m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`" | ||
|
||
/bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt | ||
m_ok "downloading diff" | ||
|
||
if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then | ||
m_error "Osmosis error" | ||
fi | ||
|
||
if [ -f $TRIM_POLY_FILE ] ; then | ||
m_ok "filtering diff" | ||
/var/lib/mod_tile/data.poly | ||
if ! $TRIM_BIN $TRIM_OPTIONS $TRIM_REGION_OPTIONS -z $CHANGE_FILE $CHANGE_FILE 1>&2 2>> "$RUNLOG"; then | ||
m_error "Trim_osc error" | ||
fi | ||
else | ||
m_ok "filtering diff skipped" | ||
fi | ||
m_ok "importing diff" | ||
#------------------------------------------------------------------------------ | ||
# Previously openstreetmap-tiles-update-expire tried to dirty layer | ||
# "$EXPIRY_MAXZOOM - 3" (which was 15) only. Instead we write all expired | ||
# tiles in range to the list (note the "-" rather than ":" in the "-e" | ||
# parameter). | ||
#------------------------------------------------------------------------------ | ||
if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then | ||
m_error "osm2pgsql error" | ||
fi | ||
|
||
#------------------------------------------------------------------------------ | ||
# The lockfile is normally removed before we expire tiles because that is | ||
# something thatcan be done in parallel with further processing. In order to | ||
# avoid rework, if actually rerendering is done rather than just deleting or | ||
# dirtying, it makes sense to move it lower down. | ||
#------------------------------------------------------------------------------ | ||
# m_ok "Import complete; removing lock file" | ||
# freelock "$LOCK_FILE" | ||
|
||
m_ok "expiring tiles" | ||
#------------------------------------------------------------------------------ | ||
# When expiring tiles we need to define the style sheet if it's not "default". | ||
# In this case it's "ajt". | ||
# Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and | ||
# $EXPIRY_MAXZOOM were dirtied. We currently re-render | ||
# tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and | ||
# delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM. | ||
# The default path to renderd.sock is fixed. | ||
#------------------------------------------------------------------------------ | ||
if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then | ||
m_info "Expiry failed" | ||
fi | ||
|
||
rm "$EXPIRY_FILE.$$" | ||
|
||
#------------------------------------------------------------------------------ | ||
# Only remove the lock file after expiry (if system is slow we want to delay | ||
# the next import, not have multiple render_expired processes running) | ||
#------------------------------------------------------------------------------ | ||
freelock "$LOCK_FILE" | ||
|
||
m_ok "Done with import" | ||
|
||
|
||
|
||
|
||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters