Skip to content
Snippets Groups Projects
Commit b5a0f44e authored by User expired's avatar User expired
Browse files

improved coloring; improved exception handling; downloading java from OTN now

parent bb13d485
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
CHANGELOG.md
+ 1
− 0
View file @ b5a0f44e
Upcoming version:
-----------------
- improved coloring; improved exception handling; downloading java from OTN now (Nikolaus Krismer)
- changes due to rename in isochrone-datamodel (Nikolaus Krismer)
- version updates (Nikolaus Krismer)
- deactivating invalid datasets (Nikolaus Krismer)
......
Vagrantfile
+ 3
− 3
View file @ b5a0f44e
......@@ -80,9 +80,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.synced_folder "log/bootstrap", "/var/log/vagrant_bootstrap", create: true, owner: "vagrant", group: "vagrant"
# Setup environment on startup (done using a shell script)
config.vm.provision "shell", path: "bootstrap/prepareEnvironment.sh", args: ["secretPhdPassword#2014!", (isLocalDeploy ? "true" : "false"), (isJenkinsDeploy ? "true" : "false")]
config.vm.provision "shell", path: "bootstrap/createDatabase.sh", args: ["niko", "secretPhdPassword#2014!"]
config.vm.provision "shell", path: "bootstrap/importData.sh", args: ["niko", "secretPhdPassword#2014!", (deployAllDatasets ? "true" : "false")]
config.vm.provision "shell", keep_color: true, path: "bootstrap/prepareEnvironment.sh", args: ["secretPhdPassword#2014!", (isLocalDeploy ? "true" : "false"), (isJenkinsDeploy ? "true" : "false")]
config.vm.provision "shell", keep_color: true, path: "bootstrap/createDatabase.sh", args: ["niko", "secretPhdPassword#2014!"]
config.vm.provision "shell", keep_color: true, path: "bootstrap/importData.sh", args: ["niko", "secretPhdPassword#2014!", (deployAllDatasets ? "true" : "false")]
# Optionally activate database logging (for queryTimeLogging this is done in the testcase itself now)
#config.vm.provision "shell", path: "bootstrap/activateDatabaseLogging.sh"
......
bootstrap/activateDatabaseLogging.sh
+ 12
− 12
View file @ b5a0f44e
......@@ -4,7 +4,7 @@
#############################
if [ "$EUID" -ne "0" ]; then
echo '- This script must be run as root!'
echo -e "\e[91mThis script must be run as root!\e[39m"
exit 1
fi
......@@ -20,7 +20,7 @@ if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
SCRIPT_DIR="/vagrant/bootstrap"
fi
if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
echo '- No variable declarations found (config.properties file not found)!'
echo -e "\e[91mNo variable declarations found (config.properties file not found)!\e[39m"
exit 1;
fi
source "${SCRIPT_DIR}/config.sh"
......@@ -29,9 +29,9 @@ source "${SCRIPT_DIR}/config.sh"
# Database logging activation #
###############################
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Enabling database logging ($(date +%H:%M:%S)):"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo " Enabling database logging ($(date +%H:%M:%S)):"
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
START=$(date +%s)
......@@ -41,22 +41,22 @@ POSTGRESQL_VERSION=$(cat ${POSTGRESQL_DIR}/PG_VERSION)
POSTGRESQL_SERVICE_NAME="postgresql-$POSTGRESQL_VERSION"
if [ -f $POSTGRESQL_GLOB_CONF ] && $AVAILABLE_POSTGRES; then
echo "Changing settings in postgresql.conf"
fn_echo "Changing settings in postgresql.conf"
# Deactivate query logging until data is imported (otherwise the import would be very slow)
sed -i 's/log_duration = off/log_duration = on/' $POSTGRESQL_GLOB_CONF
sed -i 's/log_min_duration_statement = -1/log_min_duration_statement = 0/' $POSTGRESQL_GLOB_CONF
echo "Restarting database to activate new settings"
fn_echo "Restarting database to activate new settings"
fn_service_stop "$POSTGRESQL_SERVICE_NAME"
fn_service_start "$POSTGRESQL_SERVICE_NAME"
else
echo "Not changing database logging file. No configuration file found!"
fn_echo "Not changing database logging file. No configuration file found!", "WARN"
fi
END=$(date +%s)
TOTAL=$(( $END - $START ))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Database logging enabled"
printf ' Time to enable database logging: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo " Database logging enabled"
printf '\e[92m Time to enable database logging: %dh:%dm:%ds\e[39m\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
bootstrap/config.sh
+ 58
− 14
View file @ b5a0f44e
......@@ -27,6 +27,7 @@ CACHE_DIR_REMOTE_DATA="${CACHE_DIR_REMOTE}/index.php/s/kgjm3CItQJ6P374/download?
DEPLOY_DIR="/opt"
DATA_DIR="$DEPLOY_DIR/data"
SETUP_DIR="/setup"
VERBOSE=false
WORKING_DIR="/var/log/vagrant_bootstrap"
# Parameters that can also (some must) be passed as arguments
......@@ -45,6 +46,7 @@ TOMCAT_PASSWORD="@tomcat_password@"
# Do not change anything below this line!
EXEC_POSTGRES="psql"
EXEC_WGET="wget"
POSTGRES_SHARE="$(dirname /usr/pgsql*/.)""/share"
if [ ! -d ${POSTGRES_SHARE} ]; then
POSTGRES_SHARE="/usr/share/pgsql"
......@@ -96,38 +98,79 @@ function fn_arg2string() {
VALUE="${ARG}"
fi
if [ "${VALUE:0:1}" == "@" ]; then
echo '- No (or invalid) database user supplied!'
fn_echo "No (or invalid) database user supplied!" "ERROR"
exit 1
fi
eval $OUTVAR="\$VALUE"
}
function fn_download() {
local FILE="$1"
local URL="$2"
function fn_check_status {
local status="$?"
if [ $status -ne 0 ]; then
local message="${1}"
if [ -z "${1}" ]; then
message="An error occurred. The process will exit now with status $status"
fi
if wget --no-check-certificate -q -O /dev/null "$URL"; then
wget --no-check-certificate -q -O "$FILE" "$URL"
fn_echo "${message}" "ERROR"
exit $status
fi
}
function fn_download() {
local FILE="${1}"
local URL="${2}"
local ARGS="${3} --continue --no-check-certificate --quiet"
fn_echo " - downloading ${FILE}" "DEBUG"
fn_echo " - from URL: ${URL}" "DEBUG"
fn_echo " - with wget arguments: ${ARGS}" "DEBUG"
# ${EXEC_WGET} ${ARGS} --spider "${URL}"
# fn_check_status " - download canceled, since nothing was found at URL \"${URL}\""
${EXEC_WGET} ${ARGS} --output-document="${FILE}" "${URL}"
fn_check_status " - download failed for URL \"${URL}\""
return $?
}
function fn_download_newer() {
# "wget -N" does not work with "-O" option... so we use a workaround here
local FILE="$1"
local URL="$2"
if [ -f "${1}" ]; then
fn_echo " - not downloading ${1}, since it already exists" "DEBUG"
return 0
fi
if [ ! -f "$FILE" ]; then
if wget --no-check-certificate -q -O /dev/null "$URL"; then
wget --no-check-certificate -q -O "$FILE" "$URL"
fi
fn_download "${1}" "${2}" "${3} --timestamping"
return $?
}
function fn_echo() {
local MESSAGE="${1}"
local LEVEL="${2}"
local COLOR="\e[93m" # default color is light green
local DEFAULT_COLOR="\e[39m"
case "${LEVEL,,}" in
"debug") COLOR="\e[34m" ;; # Blue
"error") COLOR="\e[91m" ;; # Light red
"warn") COLOR="\e[93m" ;; # light yellow
*) COLOR="\e[92m" ;; # light green (on level=info, no level set or invalid level given)
esac
if [ "${LEVEL,,}" = "debug" ] && ! ${VERBOSE}; then
return 0
fi
echo -e "${COLOR}${MESSAGE}${DEFAULT_COLOR}"
}
function fn_service_start() {
local SERVICE_NAME="$1"
echo "Starting service \"${SERVICE_NAME}\""
fn_echo "Starting service \"${SERVICE_NAME}\"" "DEBUG"
touch "service_start_$SERVICE_NAME.log" 2>&1
if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
service $SERVICE_NAME start >> "service_start_$SERVICE_NAME.log" 2>&1
......@@ -143,6 +186,7 @@ function fn_service_start() {
function fn_service_stop() {
local SERVICE_NAME="$1"
fn_echo "Stopping service \"${SERVICE_NAME}\"" "DEBUG"
touch "service_stop_$SERVICE_NAME.log" 2>&1
if [ "$DISTRI_TYPE" == "CentOS 6" ]; then
service $SERVICE_NAME stop >> "service_stop_$SERVICE_NAME.log" 2>&1
......
bootstrap/createDatabase.sh
+ 12
− 12
View file @ b5a0f44e
......@@ -4,7 +4,7 @@
#############################
if [ "$EUID" -ne "0" ]; then
echo '- This script must be run as root!'
echo -e "\e[91mThis script must be run as root!\e[39m"
exit 1
fi
......@@ -20,7 +20,7 @@ if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
SCRIPT_DIR="/vagrant/bootstrap"
fi
if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
echo '- No variable declarations found (config.properties file not found)!'
echo -e "\e[91mNo variable declarations found (config.properties file not found)!\e[39m"
exit 1;
fi
source "${SCRIPT_DIR}/config.sh"
......@@ -32,9 +32,9 @@ fn_arg2string PG_DB_PASSWORD "${2}" "${PG_DB_PASSWORD}"
# Database creation #
######################
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Preparing the database ($(date +%H:%M:%S)):"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo " Preparing the database ($(date +%H:%M:%S)):"
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
START=$(date +%s)
......@@ -42,7 +42,7 @@ mkdir -p $WORKING_DIR
cd $WORKING_DIR
if $AVAILABLE_POSTGRES; then
echo "Creating postgresql database \"$PG_DB_NAME\""
fn_echo "Creating postgresql database \"$PG_DB_NAME\""
sudo -u "postgres" ${EXEC_POSTGRES} -d "template1" -c "CREATE DATABASE \"$PG_DB_NAME\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "CREATE USER \"$PG_DB_USER\" WITH PASSWORD '$PG_DB_PASSWORD';" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "ALTER DATABASE \"$PG_DB_NAME\" OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1
......@@ -53,13 +53,13 @@ if $AVAILABLE_POSTGRES; then
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_POSTGRES} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/pg_procedures.sql >> $WORKING_DIR/setup_database.log 2>&1
if $AVAILABLE_POSTGIS; then
echo "Registering postgis extension for database \"$PG_DB_NAME\""
fn_echo "Registering postgis extension for database \"$PG_DB_NAME\""
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_POSTGRES} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -c "CREATE EXTENSION postgis;" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_POSTGRES} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/srid_82344_insert.sql >> $WORKING_DIR/setup_database.log 2>&1
fi
if $AVAILABLE_ISOCHRONE_DATAMODEL; then
echo "Initializing database schema for isochrone-datamodel (used for testing and datamodel creation)"
fn_echo "Initializing database schema for isochrone-datamodel (used for testing and datamodel creation)"
PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost -d "template1" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/db_create.sql" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost -d "spatial" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/schema_drop.sql" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="spatial" psql -U "spatial" -h localhost -d "spatial" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/schema_create.sql" >> $WORKING_DIR/setup_database.log 2>&1
......@@ -70,7 +70,7 @@ fi
END=$(date +%s)
TOTAL=$(( $END - $START ))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Database prepared"
printf ' Time to prepare the database: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo " Database prepared"
printf '\e[92m Time to prepare the database: %dh:%dm:%ds\e[39m\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
bootstrap/importData.sh
+ 47
− 44
View file @ b5a0f44e
#! /bin/bash
#############################
# This script imports data (e.g. from the road network of bolzano) that is
# downloaded from osm (OpenStreetMap) and then imported into available
# downloaded from OpenStreetMap (OSM) and then imported into available
# databases (neo4j_spatial, postgis and spatialite).
#############################
if [ "$EUID" -ne "0" ]; then
echo '- This script must be run as root!'
echo -e "\e[91mThis script must be run as root!\e[39m"
exit 1
fi
......@@ -22,7 +22,7 @@ if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
SCRIPT_DIR="/vagrant/bootstrap"
fi
if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
echo '- No variable declarations found (config.properties file not found)!'
echo -e "\e[91mNo variable declarations found (config.properties file not found)!\e[39m"
exit 1;
fi
source "${SCRIPT_DIR}/config.sh"
......@@ -56,29 +56,29 @@ OSM_KEYS="aerialway,highway,public_transport,railway,route"
function fn_create_synthetic_networks() {
if $AVAILABLE_ISOCHRONE_TOOLS; then
echo "Importing data for synthetic networks"
fn_echo "Importing data for synthetic networks"
# GridNetwork
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the grid network. The process will exit now with status $?"
fn_echo "An error occurred while creating the grid network. The process will exit now with status $?" "ERROR"
exit $?
fi
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d "$DENSITY" >> "$WORKING_DIR/generate_gridNetwork.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the node_density table for 'grid_s100' using isochrone-tools. The process will exit now with status $?"
fn_echo "An error occurred while creating the node_density table for 'grid_s100' using isochrone-tools. The process will exit now with status $?" "ERROR"
exit $?
fi
# SpiderNetwork
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the spider network. The process will exit now with status $?"
fn_echo "An error occurred while creating the spider network. The process will exit now with status $?" "ERROR"
exit $?
fi
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d "$DENSITY" >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the node_density table for 'spirder_l1000' using isochrone-tools. The process will exit now with status $?"
fn_echo "An error occurred while creating the node_density table for 'spider_l1000' using isochrone-tools. The process will exit now with status $?" "ERROR"
exit $?
fi
fi
......@@ -88,14 +88,14 @@ function fn_import_dataset() {
local NAME="$1"
local SRID="$2"
echo "Importing data for region of $NAME (EPSG:${SRID})"
fn_echo "Importing data for region of $NAME (EPSG:${SRID})"
if $IMPORT_DATA_POSTGIS; then
echo " - importing into postgis"
fn_echo " - importing into postgis"
fn_import_dataset_postgis "$NAME" "$SRID"
fi
if $IMPORT_DATA_NEO4J_SPATIAL; then
echo " - importing into neo4j"
fn_echo " - importing into neo4j"
fn_import_dataset_neo4j_spatial "$NAME"
fi
}
......@@ -109,7 +109,7 @@ function fn_import_dataset_neo4j_spatial() {
# We copy data from postgis for specified dataset (after postgis data import)
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.exporter.ImportDataNeo4j -t "${CITY}" > "$WORKING_DIR/import_neo4j_$CITY.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while importing data into neo4j using isochrone-tools. The process will exit now with status $?"
fn_echo "An error occurred while importing data into neo4j using isochrone-tools. The process will exit now with status $?" "ERROR"
exit $?
fi
}
......@@ -123,37 +123,37 @@ function fn_import_dataset_postgis() {
CITY=${CITY,,}
if ! $UPDATE_DATA; then
echo " - downloading SQL export"
fn_echo " - downloading SQL export"
fn_download_newer $DOWNLOAD_DIR/$SQL_EXPORT_FILE "${CACHE_DIR_REMOTE_DATA}${SQL_EXPORT_FILE}"
cp $DOWNLOAD_DIR/$SQL_EXPORT_FILE $DATA_DIR/$SQL_EXPORT_FILE >> /dev/null 2>&1
echo " - importing SQL export into PostGIS database"
fn_echo " - importing SQL export into PostGIS database"
gunzip -c $DATA_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost "$PG_DB_NAME" >> "$WORKING_DIR/import_datamodel_$CITY_$SRID.log" 2>&1
echo " - getting table permissions"
fn_echo " - getting table permissions"
tables=`PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$PG_DB_NAME"`
echo " - fixing table permissions"
fn_echo " - fixing table permissions"
for tbl in $tables; do
PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "ALTER TABLE $tbl OWNER TO $PG_DB_USER" "$PG_DB_NAME"
done
fi
if $UPDATE_DATA && [ -f "$DATA_DIR/$SQL_EXPORT_FILE" ]; then
echo " - deleting outdated SQL export (forced data update)"
fn_echo " - deleting outdated SQL export (forced data update)"
rm -rf "$DATA_DIR/$SQL_EXPORT_FILE"
fi
if $AVAILABLE_ISOCHRONE_DATAMODEL && [ ! -f "$DATA_DIR/$SQL_EXPORT_FILE" ]; then
echo " - creating datamodel using isochrone-datamodel"
fn_echo " - creating datamodel using isochrone-datamodel"
# Create datamodel using isochrone-datamodel project
DB_USERNAME="$PG_DB_USER" DB_PASSWORD="$PG_DB_PASSWORD" "$DEPLOY_DIR/isochrone-datamodel/osmPti2mmds.sh" -d -s -b -l -t${SRID} -c${CITY} >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating datamodel using isochrone-datamodel. The process will exit now with status $?"
fn_echo "An error occurred while creating datamodel using isochrone-datamodel. The process will exit now with status $?" "ERROR"
exit $?
fi
echo " - copying tables to isochrone database"
fn_echo " - copying tables to isochrone database"
# If working with multiple TARGET_SRIDs we have to delete an eventually existing nodes_density table here, so on import later a DROP TABLE ${CITY}_nodes works wihtout CASCADE (pg_dump does not use CASCADE)
PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -d isochrone -c "DROP TABLE IF EXISTS ${CITY}_nodes_density" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
PGPASSWORD="spatial" pg_dump -U spatial -h localhost -d spatial --clean --if-exists --no-privileges --no-owner -t "transformed.${CITY}_*" | sed -e "s/transformed/public/g" | PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost -d isochrone >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
......@@ -162,27 +162,30 @@ function fn_import_dataset_postgis() {
if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then
java -cp "${DEPLOY_DIR}/isochrone-tools.jar" "at.uibk.dbis.isochrone.generator.density.DensityGenerator" -t "${CITY}" -d "${DENSITY}" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the node_density table for '${CITY}' using isochrone-tools. The process will exit now with status $?"
fn_echo "An error occurred while creating the node_density table for '${CITY}' using isochrone-tools. The process will exit now with status $?" "ERROR"
exit $?
fi
fi
if $AVAILABLE_ISOCHRONE_DATAMODEL; then
# Exporting resulting database to $DATA_DIR
echo " - exporting database dump"
fn_echo " - exporting database dump"
PGPASSWORD="$PG_DB_PASSWORD" pg_dump -U "$PG_DB_USER" -h localhost -d isochrone --clean --compress=5 --if-exists -t "${CITY}_*" --file="$DATA_DIR/$SQL_EXPORT_FILE" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
fi
}
function fn_init_geoserver() {
# TODO: Do we really want to import data from OSM, so that it is available in geoserver (we publish data from within isochron-web to geoserver, but why should we do that with the raw data -> pgRouing comparison or similar?)
# TODO: Needs testing
# TODO: Do we really want to import data from OSM, so that it is available in geoserver?
# (we publish data from within isochrone-web to geoserver, but why should we do that with the raw data -> pgRouing comparison or similar?)
# Needs testing !!!
if $AVAILABLE_NEO4J_SPATIAL; then
echo "Configuring neo4j data permissions for geoserver"
fn_echo "Configuring neo4j data permissions for geoserver"
chown -R $TOMCAT_USER:$TOMCAT_USER $DEPLOY_DIR/neo4j/data/graph.db
fi
echo "Preparing data from OpenStreetMap to be imported into geoserver"
fn_echo "Preparing data from OpenStreetMap to be imported into geoserver"
fn_filter_osm_data "Bolzano" "${OSM_FILE_ITALY}"
fn_filter_osm_data "Innsbruck" "${OSM_FILE_AUSTRIA}"
fn_filter_osm_data "Salzburg" "${OSM_FILE_AUSTRIA}"
......@@ -193,9 +196,9 @@ function fn_init_geoserver() {
fn_filter_osm_data "AltoAdige" "${OSM_FILE_ITALY}"
fi
# TODO: We need to import the data into the database before issuing to geoserver that there are tables available
# TODO: We need to import the data into the database before issueing to geoserver that there are tables available
echo "Configuring geoserver $GEOSERVER_VERSION using geoserver-shell $GEOSERVER_SHELL_VERSION"
fn_echo "Configuring geoserver $GEOSERVER_VERSION using geoserver-shell $GEOSERVER_SHELL_VERSION"
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_ws.gs >> setup_geoserver_workspace.log 2>&1
sh $DEPLOY_DIR/gs-shell/bin/gs-shell --cmdfile $SHARED_CONF_DIR/geoserver_setup_styles.gs >> setup_geoserver_styles.log 2>&1
cp $SHARED_IMG_DIR/* $DEPLOY_DIR/geoserver/data/styles >> setup_geoserver_styles.log 2>&1
......@@ -236,27 +239,27 @@ function fn_import_spatialite() {
SPATIALITE_FILENAME="isochrone_${SRID}.spatialite"
if ! $UPDATE_DATA; then
echo "Importing spatialite database"
fn_echo "Importing spatialite database"
fn_download_newer $DOWNLOAD_DIR/$SPATIALITE_FILENAME "${CACHE_DIR_REMOTE_DATA}${SPATIALITE_FILENAME}"
cp -f "$DOWNLOAD_DIR/$SPATIALITE_FILENAME" "$DATA_DIR/"
fi
if $UPDATE_DATA && [ -f "$DATA_DIR/$SPATIALITE_FILENAME" ]; then
echo "Deleting outdated spatialite database (forced data update)"
fn_echo "Deleting outdated spatialite database (forced data update)"
rm -rf "$DATA_DIR/$SPATIALITE_FILENAME"
fi
if $AVAILABLE_GDAL && [ ! -f "$DATA_DIR/$SPATIALITE_FILENAME" ]; then
echo "Exporting PostGIS data into spatialite database"
fn_echo "Exporting PostGIS data into spatialite database"
$DEPLOY_DIR/gdal/apps/ogr2ogr --config PG_LIST_ALL_TABLES YES --config PG_SKIP_VIEWS YES -progress -f "SQLite" "$DATA_DIR/$SPATIALITE_FILENAME" PG:"host=localhost dbname=isochrone user=$PG_DB_USER password=$PG_DB_PASSWORD" -lco LAUNDER=yes -dsco SPATIALITE=yes -lco SPATIAL_INDEX=yes -gt 65536 >> "$WORKING_DIR/import_spatialite.log" 2>&1
fi
if [ -f "$DATA_DIR/$SPATIALITE_FILENAME" ]; then
echo " - setting spatialite database permissions"
fn_echo " - setting spatialite database permissions"
chown -R apache:apache "$DATA_DIR/$SPATIALITE_FILENAME"
chmod -R -g+rwX "$DATA_DIR/$SPATIALITE_FILENAME"
else
echo " - spatialite database could not be created!" >&2
fn_echo " - spatialite database could not be created!" >&2
fi
}
......@@ -269,25 +272,25 @@ function fn_filter_osm_data() {
local OSM_FILE_FILTERED="${CITY}_filtered.osm"
if $UPDATE_DATA && [ -f "$DATA_DIR/$OSM_FILE_FILTERED" ]; then
echo " - deleting outdated osm export (forced data update)"
fn_echo " - deleting outdated osm export (forced data update)"
rm -rf "$DATA_DIR/$OSM_FILE_FILTERED"
fi
if [ ! -f "$DATA_DIR/$OSM_FILE_FILTERED" ]; then
echo " - downloading cached OpenStreetMap file"
fn_echo " - downloading cached OpenStreetMap file"
fn_download_newer $DOWNLOAD_DIR/$OSM_FILE_FILTERED $CACHE_DIR_REMOTE_DATA/$OSM_FILE_FILTERED
cp $DOWNLOAD_DIR/$OSM_FILE_FILTERED $DATA_DIR/$OSM_FILE_FILTERED >> /dev/null 2>&1
fi
if [ ! -f $DATA_DIR/$OSM_FILE_FILTERED ]; then
echo " - downloading OpenStreetMap file (no cached file found)"
fn_echo " - downloading OpenStreetMap file (no cached file found)"
OSM_FILENAME=$DOWNLOAD_DIR/${OSM_FILE##*/}
BOUNDING="${CITY_POLYGON_DIR}/${city}.poly"
fn_download_newer $OSM_FILENAME $OSM_DOWNLOAD_MIRROR/$OSM_FILE
if [[ "$BOUNDING" == *.poly ]]; then
echo " - filtering OpenStreetMap file"
fn_echo " - filtering OpenStreetMap file"
$DEPLOY_DIR/osmosis/bin/osmosis --read-pbf file="$OSM_FILENAME" --bounding-polygon file="$BOUNDING_POLYGON" --way-key keyList="$OSM_KEYS" --used-node --write-xml file="$DATA_DIR/$OSM_FILE_FILTERED" >> "filter_osmData_$NAME.log" 2>&1
fi
fi
......@@ -299,9 +302,9 @@ function fn_filter_osm_data() {
################
if $DEPLOY_ANY_DATASET; then
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Importing datasets ($(date +%H:%M:%S)):"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo " Importing datasets ($(date +%H:%M:%S)):"
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
START=$(date +%s)
......@@ -339,8 +342,8 @@ if $DEPLOY_ANY_DATASET; then
END=$(date +%s)
TOTAL=$(( $END - $START ))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Datasets imported"
printf ' Time to import the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fn_echo " Datasets imported"
printf '\e[92m Time to import the datasets: %dh:%dm:%ds\e[39m\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
fn_echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fi
bootstrap/prepareEnvironment.sh
+ 157
− 145
View file @ b5a0f44e
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment