-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathcall_pyosmium.sh
executable file
·155 lines (154 loc) · 6.66 KB
/
call_pyosmium.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
#!/bin/bash
# -----------------------------------------------------------------------------
# call_pyosmium.sh
#
# Copyright (C) 2018-2025 Andy Townsend
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# -----------------------------------------------------------------------------
# Designed to be run from cron, as the user that owns the database (normally "_renderd").
# Note that "render_expired" doesn't use TILEDIR from map name in /etc/renderd.conf
# when deciding if tiles exist already; it assumes they must be below /var/cache .
# It therefore makes sense to use /var/cache/renderd/pyosmium.gis for the files needed here.
#
# First, define the local user account we are using.
# "local_filesystem_user" is whichever non-root account is used to fetch from
# github.
#
local_filesystem_user=ajtown
#
# To initialise "sequence.state", run:
#
# sudo mkdir /var/cache/renderd/pyosmium.gis
# sudo chown _renderd /var/cache/renderd/pyosmium.gis
# cd /var/cache/renderd/pyosmium.gis
# sudo -u _renderd pyosmium-get-changes -D 2022-06-08T20:21:25Z -f sequence.state -v
#
# with an appropriate date.
#
if [[ ! -f /var/cache/renderd/pyosmium.gis/sequence.state ]]
then
echo "/var/cache/renderd/pyosmium.gis/sequence.state does not exist"
exit 1
fi
#
if ! command -v pyosmium-get-changes &> /dev/null
then
echo "pyosmium-get-changes could not be found"
exit 1
fi
#
if ! command -v osm2pgsql &> /dev/null
then
echo "osm2pgsql could not be found"
exit 1
fi
#
if ! command -v render_expired &> /dev/null
then
echo "render_expired could not be found"
exit 1
fi
#
if [[ -f /var/cache/renderd/pyosmium.gis/call_pyosmium.running ]]
then
echo "call_pyosmium already running; /var/cache/renderd/pyosmium.gis/call_pyosmium.running exists"
exit 1
else
touch /var/cache/renderd/pyosmium.gis/call_pyosmium.running
fi
#
echo
echo "Pyosmium update started: " `date`
#
cd /var/cache/renderd/pyosmium.gis/
rm newchange.osc.gz > pyosmium.$$ 2>&1
cp sequence.state sequence.state.old
#------------------------------------------------------------------------------
# "-s 20" here means "get 20MB at once".
# The value can be adjusted up or down as needed.
#------------------------------------------------------------------------------
#
pyosmium-get-changes -f sequence.state -o newchange.osc.gz -s 20 >> pyosmium.$$ 2>&1
#
#------------------------------------------------------------------------------
# Trim the downloaded changes to only the ones that apply to our region.
#
# When using trim_osc.py we can define either a bounding box (such as this
# example for England and Wales) or a polygon.
# See https://github.com/zverik/regional .
# This area will usually correspond to the data originally loaded.
#------------------------------------------------------------------------------
TRIM_BIN=/home/${local_filesystem_user}/src/regional/trim_osc.py
TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27"
#TRIM_REGION_OPTIONS="-p region.poly"
if [[ -f $TRIM_BIN ]]
then
echo "Filtering newchange.osc.gz"
if ! $TRIM_BIN -d gis $TRIM_REGION_OPTIONS -z newchange.osc.gz newchange.osc.gz > trim.$$ 2>&1
then
echo "Trim_osc error but continue anyway"
fi
else
echo "${TRIM_BIN} does not exist but continue anyway"
fi
#
#------------------------------------------------------------------------------
# The osm2pgsql append line will need to be tailored to match the running system (memory and number of processors), the style in use, and
# the number of zoom levels to write dirty tiles for.
#------------------------------------------------------------------------------
echo "Importing newchange.osc.gz"
if ! osm2pgsql --append --slim -d gis -C 2500 --number-processes 2 --multi-geometry --tag-transform-script /home/${local_filesystem_user}/src/SomeoneElse-style/style.lua -S /home/${local_filesystem_user}/src/openstreetmap-carto-AJT/openstreetmap-carto.style --expire-tiles=1-20 --expire-output=/var/cache/renderd/pyosmium.gis/dirty_tiles.txt /var/cache/renderd/pyosmium.gis/newchange.osc.gz > osm2pgsql.$$ 2>&1
then
# ------------------------------------------------------------------------------
# The osm2pgsql import failed; show the error, revert to the previous import
# sequence and remove the "running" flag to try again.
# Don't delete the command output files to allow later investigation.
# ------------------------------------------------------------------------------
echo "osm2pgsql append error"
cat osm2pgsql.$$
cp sequence.state.old sequence.state
rm /var/cache/renderd/pyosmium.gis/call_pyosmium.running
exit 1
else
tail -1 osm2pgsql.$$
fi
#
#------------------------------------------------------------------------------
# This line is exactly the same as the "expire_tiles.sh"
# that would be used with "update_tiles.sh" (which calls "osm2pgsql-replication update" rather than the more flexible "pyosmium-get-changes")
# The arguments can be tailored to do different things at different zoom levels as desired.
#------------------------------------------------------------------------------
echo "Expiring tiles"
render_expired --map=ajt --min-zoom=13 --touch-from=13 --delete-from=19 --max-zoom=20 -s /run/renderd/renderd.sock < /var/cache/renderd/pyosmium.gis/dirty_tiles.txt > render_expired.$$ 2>&1
tail -9 render_expired.$$
#
#------------------------------------------------------------------------------
# If debugging tile dirtying, comment out these two lines and uncomment the third.
#------------------------------------------------------------------------------
rm /var/cache/renderd/pyosmium.gis/dirty_tiles.txt >> pyosmium.$$ 2>&1
rm render_expired.$$
#mv /var/cache/renderd/pyosmium.gis/dirty_tiles.txt /var/cache/renderd/pyosmium.gis/dirty_tiles.txt.$$ >> pyosmium.$$ 2>&1
#
#------------------------------------------------------------------------------
# Tidy up files containing output from each command and the file that shows
# that the script is running
#------------------------------------------------------------------------------
rm trim.$$
rm osm2pgsql.$$
rm pyosmium.$$
rm /var/cache/renderd/pyosmium.gis/call_pyosmium.running
#
echo "Database Replication Lag:" `pyosmium_replag.sh -h`
#