Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
#! /bin/bash
#############################
# This script imports data (e.g. from the road network of bolzano) that is
# downloaded from osm (OpenStreetMap) and then imported into available
# databases (neo4j and postgresql).
#############################
if [ "$EUID" -ne "0" ]; then
echo '- This script must be run as root!'
exit 1
fi
############################
# Variable definitions #
############################
SCRIPT="$(readlink -f ${BASH_SOURCE[0]})"
SCRIPT_DIR="$(dirname ${SCRIPT})"
if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
# If config.properties is not besides the bash-script (most likely because vagrant uploaded it into the guest)
# we will try to find it in the shared folder
SCRIPT_DIR="/vagrant/bootstrap"
fi
if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
echo '- No variable declarations found (config.properties file not found)!'
exit 1;
fi
source "${SCRIPT_DIR}/config.sh"
if ! $IMPORT_DATA_OSM && ! $IMPORT_DATA_OSM_CACHED && ! $IMPORT_DATA_SQL; then
exit 0
fi
fn_arg2string PG_DB_USER "${1}" "${PG_DB_USER}"
fn_arg2string PG_DB_PASSWORD "${2}" "${PG_DB_PASSWORD}"
fn_arg2boolean IS_LOCAL_TEST_DEPLOY "${3}" "${IS_LOCAL_TEST_DEPLOY}"
############################
# Function definitions #
############################
fn_import_data() {
local NAME="$1"
local OSM_FILE="$2"
local OSM_FILE_FILTERED="$3"
local BOUNDING="$4"
local SQL_EXPORT_FILE="$5"
local TABLE_PREFIX="$6"
local VALID_OSM=false
local VALID_SQL=false
if $IMPORT_DATA_SQL && $AVAILABLE_POSTGIS && $IMPORT_SCHEDULES; then
if [ -n "$SQL_EXPORT_FILE" ]; then
VALID_SQL=true
fi
fi
if ( $AVAILABLE_PGROUTING || $AVAILABLE_NEO4J_SPATIAL ) && ( $IMPORT_DATA_OSM_CACHED || $IMPORT_DATA_OSM ); then
# osm data does not need to be downloaded (since it would not be imported afterwards)
VALID_OSM=true
fi
echo "Importing data for region of $NAME"
if ! ( $VALID_SQL || $VALID_OSM ); then
echo " - skipping import"
return 1
fi
if $VALID_SQL; then
echo " - downloading sql export"
fn_download_newer $DOWNLOAD_DIR/$SQL_EXPORT_FILE $CACHE_DIR_REMOTE/$SQL_EXPORT_FILE
cp $DOWNLOAD_DIR/$SQL_EXPORT_FILE $DEPLOY_DIR/$SQL_EXPORT_FILE >> /dev/null 2>&1
echo " - importing sql export into postgis database"
gunzip -c $DEPLOY_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost "$PG_DB_NAME" >> "$WORKING_DIR/import_schedule_$NAME.log" 2>&1
echo " - getting table permissions"
tables=`PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$PG_DB_NAME"`
echo " - fixing table permissions"
for tbl in $tables ; do
PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "ALTER TABLE $tbl OWNER TO $PG_DB_USER" "$PG_DB_NAME";
done
fi
if ! $VALID_OSM; then
return 0
fi
if $IMPORT_DATA_OSM_CACHED; then
echo " - downloading cached OpenStreetMap file"
fn_download_newer $DOWNLOAD_DIR/$OSM_FILE_FILTERED $CACHE_DIR_REMOTE/$OSM_FILE_FILTERED
cp $DOWNLOAD_DIR/$OSM_FILE_FILTERED $DEPLOY_DIR/$OSM_FILE_FILTERED >> /dev/null 2>&1
fi
if [ ! -f $DEPLOY_DIR/$OSM_FILE_FILTERED ]; then
if $IMPORT_DATA_OSM; then
echo " - downloading OpenStreetMap file (no cached file found)"
OSM_FILENAME=$DOWNLOAD_DIR/${OSM_FILE##*/}
fn_download_newer $OSM_FILENAME $OSM_DOWNLOAD_MIRROR/$OSM_FILE
echo " - filtering OpenStreetMap data"
if [[ "$BOUNDING" == *.poly ]]; then
$DEPLOY_DIR/osmosis/bin/osmosis --read-pbf file="$OSM_FILENAME" --bounding-polygon file="$BOUNDING" --way-key keyList="$OSM_KEYS" --used-node --write-xml file="$DEPLOY_DIR/$OSM_FILE_FILTERED" >> "initialize_osmData_$NAME.log" 2>&1
else
$DEPLOY_DIR/osmosis/bin/osmosis --read-pbf file="$OSM_FILENAME" --bounding-box $BOUNDING --way-key keyList="$OSM_KEYS" --used-node --write-xml file="$DEPLOY_DIR/$OSM_FILE_FILTERED" >> "initialize_osmData_$NAME.log" 2>&1
fi
else
echo " - problems with downloading cached osm file... this will most likely lead cause errors"
fi
fi
if $AVAILABLE_PGROUTING; then
echo " - importing filtered OpenStreetMap data into pgRouting"
$DEPLOY_DIR/osm2pgrouting/build/osm2pgrouting -file "$DEPLOY_DIR/$OSM_FILE_FILTERED" -conf "$OSM2PGROUTING_MAPPING_CONF" -dbname $PG_DB_NAME -user $PG_DB_USER -passwd $PG_DB_PASSWORD -clean -prefixtables $TABLE_PREFIX >> "import_osm2pgrouting_$NAME.log" 2>&1
echo " - creating pgRouting topology"
PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -c "SELECT pgr_createTopology('${TABLE_PREFIX}ways', $TOPOLOGY_TOLERANCE, 'the_geom', 'gid');" >> "$WORKING_DIR/setup_database_$NAME.log" 2>&1
fi
if $AVAILABLE_NEO4J_SPATIAL; then
echo " - importing filtered OpenStreetMap data into neo4j-spatial"
cd $DEPLOY_DIR/neo4j-spatial
mvn exec:java -Dexec.mainClass=org.neo4j.gis.spatial.osm.OSMImporter -Dexec.args="$DEPLOY_DIR/neo4j/data/graph.db $DEPLOY_DIR/$OSM_FILE_FILTERED" >> $WORKING_DIR/import_osm2neo4j.log 2>&1
cd $WORKING_DIR
fi
return 0
}
################
# Data import #
################
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Importing datasets ($(date +%H:%M:%S)):"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
START=$(date +%s)
# Import real world datasets
fn_import_data "Bozen" "$OSM_FILE_ITALY" "$OSM_FILE_BZ_FILTERED" "$OSM_BBOX_BZ" "bz_export.sql.gz" "$TABLE_PREFIX_BZ"
fn_import_data "Innsbruck" "$OSM_FILE_AUSTRIA" "$OSM_FILE_IBK_FILTERED" "$OSM_BBOX_IBK" "" "$TABLE_PREFIX_IBK"
fn_import_data "San Francisco" "$OSM_FILE_CALIFORNIA" "$OSM_FILE_SF_FILTERED" "$OSM_BBOX_SF" "sf_export.sql.gz" $TABLE_PREFIX_SF
#if ! $IS_LOCAL_TEST_DEPLOY; then
# fn_import_data "Italy" "$OSM_FILE_ITALY" "$OSM_FILE_IT_FILTERED" "$OSM_BBOX_IT" "it_export.sql.gz" $TABLE_PREFIX_IT
# fn_import_data "Alto Adige" "$OSM_FILE_ITALY" "$OSM_FILE_ST_FILTERED" "$OSM_BBOX_ST" "st_export.sql.gz" $TABLE_PREFIX_ST
#fi
# Generate synthetic data
if $AVAILABLE_POSTGIS && $INSTALL_ISOCHRONE_TOOLS; then
java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1
java -cp $DOWNLOAD_DIR/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
fi
if $AVAILABLE_GEOSERVER; then
if $AVAILABLE_NEO4J_SPATIAL; then
echo "Configuring neo4j data permissions for geoserver"
chown -R $TOMCAT_USER:$TOMCAT_USER $DEPLOY_DIR/neo4j/data/graph.db
fi
echo "Configuring geoserver $GEOSERVER_VERSION using geoserver-shell $GEOSERVER_SHELL_VERSION"
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_ws.gs >> setup_geoserver_workspace.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_styles.gs >> setup_geoserver_styles.log 2>&1
cp $SHARED_IMG_DIR/* $DEPLOY_DIR/geoserver/data/styles >> setup_geoserver_styles.log 2>&1
if $AVAILABLE_NEO4J_SPATIAL; then
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_neo4j_ds.gs >> setup_geoserver_neo4j.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_neo4j_ft_bz.gs >> setup_geoserver_neo4j.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_neo4j_ft_ibk.gs >> setup_geoserver_neo4j.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_neo4j_ft_sf.gs >> setup_geoserver_neo4j.log 2>&1
if ! $IS_LOCAL_TEST_DEPLOY; then
# sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_neo4j_ft_it.gs >> setup_geoserver_neo4j.log 2>&1
# sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_neo4j_ft_st.gs >> setup_geoserver_neo4j.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_neo4j_ft.gs >> setup_geoserver_neo4j.log 2>&1
fi
fi
if $AVAILABLE_POSTGIS; then
# copy geoserver_setup_postgis_ds and replace username/password
cp $SHARED_CONF_DIR/geoserver_setup_postgis_ds.gs $WORKING_DIR/geoserver_setup_postgis_ds.gs
sed -i "s/@db_username@/$PG_DB_USER/" $WORKING_DIR/geoserver_setup_postgis_ds.gs
sed -i "s/@db_password@/$PG_DB_PASSWORD/" $WORKING_DIR/geoserver_setup_postgis_ds.gs
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $WORKING_DIR/geoserver_setup_postgis_ds.gs >> setup_geoserver_postgis.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_postgis_ft_bz.gs >> setup_geoserver_postgis.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_postgis_ft_ibk.gs >> setup_geoserver_postgis.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_postgis_ft_sf.gs >> setup_geoserver_postgis.log 2>&1
if ! $IS_LOCAL_TEST_DEPLOY; then
# sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_postgis_ft_it.gs >> setup_geoserver_postgis.log 2>&1
# sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_postgis_ft_st.gs >> setup_geoserver_postgis.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_postgis_ft.gs >> setup_geoserver_postgis.log 2>&1
fi
fi
fi
END=$(date +%s)
TOTAL=$(( $END - $START ))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Datasets imported"
printf 'Time to import the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"