Skip to content
Snippets Groups Projects
Commit 45b4b6e5 authored by User expired's avatar User expired
Browse files

moved database creation script to isochrone-datamodel

parent 596903e3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
CHANGELOG.md
+ 1
− 0
View file @ 45b4b6e5
Upcoming version:
-----------------
- moved database creation script to isochrone-datamodel (Nikolaus Krismer)
- removing remote cache for geoserver-shell (Nikolaus Krismer)
- improved coloring; improved exception handling; downloading java from OTN now (Nikolaus Krismer)
- changes due to rename in isochrone-datamodel (Nikolaus Krismer)
......
bootstrap/config.sh
+ 27
− 14
View file @ 45b4b6e5
......@@ -25,6 +25,8 @@ CACHE_DIR_REMOTE="https://dbis-owncloud.uibk.ac.at/index.php/s/kgjm3CItQJ6P374/d
DEPLOY_DIR="/opt"
DATA_DIR="$DEPLOY_DIR/data"
SETUP_DIR="/setup"
USE_COLOR=true
USE_TIMINGS=false
VERBOSE=false
WORKING_DIR="/var/log/vagrant_bootstrap"
......@@ -43,8 +45,10 @@ TOMCAT_PASSWORD="@tomcat_password@"
# Do not change anything below this line!
EXEC_POSTGRES="psql"
EXEC_WGET="wget"
EXEC_JAVA=java
EXEC_PGDUMP=pg_dump
EXEC_PSQL=psql
EXEC_WGET=wget
POSTGRES_SHARE="$(dirname /usr/pgsql*/.)""/share"
if [ ! -d ${POSTGRES_SHARE} ]; then
POSTGRES_SHARE="/usr/share/pgsql"
......@@ -56,7 +60,7 @@ AVAILABLE_ISOCHRONE_DATAMODEL=$([ -d "${DEPLOY_DIR}/isochrone-datamodel" ] && ec
AVAILABLE_ISOCHRONE_TOOLS=$([ -f "${DEPLOY_DIR}/isochrone-tools.jar" ] && echo true || echo false)
AVAILABLE_NEO4J=$([ -d "${DEPLOY_DIR}/neo4j" ] && echo true || echo false)
AVAILABLE_NEO4J_SPATIAL=$([ -d "${DEPLOY_DIR}/neo4j-spatial" ] && echo true || echo false)
AVAILABLE_POSTGRES=$(type ${EXEC_POSTGRES} >/dev/null 2>&1 && echo true || echo false)
AVAILABLE_POSTGRES=$(type ${EXEC_PSQL} >/dev/null 2>&1 && echo true || echo false)
AVAILABLE_POSTGIS=$([ -f "${POSTGRES_SHARE}/extension/postgis.control" ] && echo true || echo false)
AVAILABLE_SPATIALITE_ADMIN=$([ -f "${DEPLOY_DIR}/phpliteadmin/phpliteadmin.config.php" ] && echo true || echo false)
......@@ -151,18 +155,27 @@ function fn_echo() {
local COLOR="\e[93m" # default color is light green
local DEFAULT_COLOR="\e[39m"
case "${LEVEL,,}" in
"debug") COLOR="\e[34m" ;; # Blue
"error") COLOR="\e[91m" ;; # Light red
"warn") COLOR="\e[93m" ;; # light yellow
*) COLOR="\e[92m" ;; # light green (on level=info, no level set or invalid level given)
esac
if [ "${LEVEL,,}" = "debug" ] && ! ${VERBOSE}; then
return 0
if ${USE_TIMINGS}; then
TIME=$(date +"%T.%N")
MESSAGE="[${TIME}] ${MESSAGE}"
fi
if ${USE_COLOR} ; then
case "${LEVEL,,}" in
"debug") COLOR="\e[34m" ;; # Blue
"error") COLOR="\e[91m" ;; # Light red
"warn") COLOR="\e[93m" ;; # light yellow
*) COLOR="\e[92m" ;; # light green (on level=info, no level set or invalid level given)
esac
if [ "${LEVEL,,}" = "debug" ] && ! ${VERBOSE}; then
return 0
fi
echo -e "${COLOR}${MESSAGE}${DEFAULT_COLOR}"
else
echo "${MESSAGE}"
fi
echo -e "${COLOR}${MESSAGE}${DEFAULT_COLOR}"
}
function fn_service_start() {
......
bootstrap/createDatabase.sh
+ 9
− 9
View file @ 45b4b6e5
......@@ -43,19 +43,19 @@ cd $WORKING_DIR
if $AVAILABLE_POSTGRES; then
fn_echo "Creating postgresql database \"$PG_DB_NAME\""
sudo -u "postgres" ${EXEC_POSTGRES} -d "template1" -c "CREATE DATABASE \"$PG_DB_NAME\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "CREATE USER \"$PG_DB_USER\" WITH PASSWORD '$PG_DB_PASSWORD';" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "ALTER DATABASE \"$PG_DB_NAME\" OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "GRANT ALL PRIVILEGES ON DATABASE \"$PG_DB_NAME\" TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "ALTER SCHEMA public OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "ALTER ROLE \"$PG_DB_USER\" SUPERUSER;" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "template1" -c "CREATE DATABASE \"$PG_DB_NAME\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "CREATE USER \"$PG_DB_USER\" WITH PASSWORD '$PG_DB_PASSWORD';" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "ALTER DATABASE \"$PG_DB_NAME\" OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "GRANT ALL PRIVILEGES ON DATABASE \"$PG_DB_NAME\" TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "ALTER SCHEMA public OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "ALTER ROLE \"$PG_DB_USER\" SUPERUSER;" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_POSTGRES} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/pg_procedures.sql >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/pg_procedures.sql >> $WORKING_DIR/setup_database.log 2>&1
if $AVAILABLE_POSTGIS; then
fn_echo "Registering postgis extension for database \"$PG_DB_NAME\""
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_POSTGRES} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -c "CREATE EXTENSION postgis;" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_POSTGRES} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/srid_82344_insert.sql >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -c "CREATE EXTENSION postgis;" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/srid_82344_insert.sql >> $WORKING_DIR/setup_database.log 2>&1
fi
if $AVAILABLE_ISOCHRONE_DATAMODEL; then
......
bootstrap/importData.sh
+ 13
− 13
View file @ 45b4b6e5
......@@ -59,24 +59,24 @@ function fn_create_synthetic_networks() {
fn_echo "Importing data for synthetic networks"
# GridNetwork
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1
${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1
if [ $? -ne 0 ]; then
fn_echo "An error occurred while creating the grid network. The process will exit now with status $?" "ERROR"
exit $?
fi
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d "$DENSITY" >> "$WORKING_DIR/generate_gridNetwork.log" 2>&1
${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d "$DENSITY" >> "$WORKING_DIR/generate_gridNetwork.log" 2>&1
if [ $? -ne 0 ]; then
fn_echo "An error occurred while creating the node_density table for 'grid_s100' using isochrone-tools. The process will exit now with status $?" "ERROR"
exit $?
fi
# SpiderNetwork
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
if [ $? -ne 0 ]; then
fn_echo "An error occurred while creating the spider network. The process will exit now with status $?" "ERROR"
exit $?
fi
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d "$DENSITY" >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d "$DENSITY" >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
if [ $? -ne 0 ]; then
fn_echo "An error occurred while creating the node_density table for 'spider_l1000' using isochrone-tools. The process will exit now with status $?" "ERROR"
exit $?
......@@ -107,7 +107,7 @@ function fn_import_dataset_neo4j_spatial() {
CITY=${CITY,,}
# We copy data from postgis for specified dataset (after postgis data import)
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.exporter.ImportDataNeo4j -t "${CITY}" > "$WORKING_DIR/import_neo4j_$CITY.log" 2>&1
${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.exporter.ImportDataNeo4j -t "${CITY}" > "$WORKING_DIR/import_neo4j_$CITY.log" 2>&1
if [ $? -ne 0 ]; then
fn_echo "An error occurred while importing data into neo4j using isochrone-tools. The process will exit now with status $?" "ERROR"
exit $?
......@@ -128,14 +128,14 @@ function fn_import_dataset_postgis() {
cp $DOWNLOAD_DIR/$SQL_EXPORT_FILE $DATA_DIR/$SQL_EXPORT_FILE >> /dev/null 2>&1
fn_echo " - importing SQL export into PostGIS database"
gunzip -c $DATA_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost "$PG_DB_NAME" >> "$WORKING_DIR/import_datamodel_$CITY_$SRID.log" 2>&1
gunzip -c $DATA_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost "$PG_DB_NAME" >> "$WORKING_DIR/import_datamodel_$CITY_$SRID.log" 2>&1
fn_echo " - getting table permissions"
tables=`PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$PG_DB_NAME"`
tables=`PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$PG_DB_NAME"`
fn_echo " - fixing table permissions"
for tbl in $tables; do
PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "ALTER TABLE $tbl OWNER TO $PG_DB_USER" "$PG_DB_NAME"
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -c "ALTER TABLE $tbl OWNER TO $PG_DB_USER" "$PG_DB_NAME"
done
fi
......@@ -154,13 +154,13 @@ function fn_import_dataset_postgis() {
fi
fn_echo " - copying tables to isochrone database"
# If working with multiple TARGET_SRIDs we have to delete an eventually existing nodes_density table here, so on import later a DROP TABLE ${CITY}_nodes works wihtout CASCADE (pg_dump does not use CASCADE)
PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -d isochrone -c "DROP TABLE IF EXISTS ${CITY}_nodes_density" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
PGPASSWORD="spatial" pg_dump -U spatial -h localhost -d spatial --clean --if-exists --no-privileges --no-owner -t "transformed.${CITY}_*" | sed -e "s/transformed/public/g" | PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost -d isochrone >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
# If working with multiple TARGET_SRIDs we have to delete an eventually existing nodes_density table here, so on import later a DROP TABLE ${CITY}_nodes works wihtout CASCADE (${EXEC_PGDUMP} does not use CASCADE)
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -d isochrone -c "DROP TABLE IF EXISTS ${CITY}_nodes_density" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
PGPASSWORD="spatial" ${EXEC_PGDUMP} -U spatial -h localhost -d spatial --clean --if-exists --no-privileges --no-owner -t "transformed.${CITY}_*" | sed -e "s/transformed/public/g" | PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -d isochrone >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
fi
if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then
java -cp "${DEPLOY_DIR}/isochrone-tools.jar" "at.uibk.dbis.isochrone.generator.density.DensityGenerator" -t "${CITY}" -d "${DENSITY}" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
${EXEC_JAVA} -cp "${DEPLOY_DIR}/isochrone-tools.jar" "at.uibk.dbis.isochrone.generator.density.DensityGenerator" -t "${CITY}" -d "${DENSITY}" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
if [ $? -ne 0 ]; then
fn_echo "An error occurred while creating the node_density table for '${CITY}' using isochrone-tools. The process will exit now with status $?" "ERROR"
exit $?
......@@ -170,7 +170,7 @@ function fn_import_dataset_postgis() {
if $AVAILABLE_ISOCHRONE_DATAMODEL; then
# Exporting resulting database to $DATA_DIR
fn_echo " - exporting database dump"
PGPASSWORD="$PG_DB_PASSWORD" pg_dump -U "$PG_DB_USER" -h localhost -d isochrone --clean --compress=5 --if-exists -t "${CITY}_*" --file="$DATA_DIR/$SQL_EXPORT_FILE" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PGDUMP} -U "$PG_DB_USER" -h localhost -d isochrone --clean --compress=5 --if-exists -t "${CITY}_*" --file="$DATA_DIR/$SQL_EXPORT_FILE" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
fi
}
......
etc/.gitignore deleted 100644 → 0
+ 0
− 1
View file @ 596903e3
*.log
\ No newline at end of file
etc/createDataset.sh deleted 100755 → 0
+ 0
− 110
View file @ 596903e3
#! /bin/bash
#############################
# This script creates datasets (e.g. from the road network of bolzano)
# from data that is downloaded from osm (OpenStreetMap) and then imported
# into the postgis database.
#
# In difference to importData.sh this only creates the given datasets with the
# given SRIDs by impoering them into postgis.
#
# Note that you need some synlinks in directoy /opt/ (to isochrone-datamodel
# and isochrone-tools project) for this script to work correctly.
#############################
if [ "$EUID" -ne "0" ]; then
echo '- This script must be run as root!'
exit 1
fi
########################################################################
# Config import (so we can override some things like working directory #
########################################################################
SCRIPT="$(readlink -f ${BASH_SOURCE[0]})"
CURRENT_DIR="$(dirname ${SCRIPT})"
SCRIPT_DIR="${CURRENT_DIR}/../bootstrap"
if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
# If config.properties is not besides the bash-script (most likely because vagrant uploaded it into the guest)
# we will try to find it in the shared folder
SCRIPT_DIR="/vagrant/bootstrap"
fi
if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
echo '- No variable declarations found (config.properties file not found)!'
exit 1;
fi
source "${SCRIPT_DIR}/config.sh"
############################
# Precondition checks #
############################
if ! $AVAILABLE_ISOCHRONE_DATAMODEL; then
echo " - Project folder isochrone-datamodel not present in directory '${DEPLOY_DIR}'"
exit 1
fi
if ! $AVAILABLE_ISOCHRONE_TOOLS; then
echo " - Project archive isochrone-tools not present in directory '${DEPLOY_DIR}'"
exit 1
fi
if ! $AVAILABLE_POSTGRES; then
echo " - PostgreSQL database not installed on this system"
exit 1
fi
if ! $AVAILABLE_POSTGIS; then
echo " - Postgis extension not installed on this system"
exit 1
fi
############################
# Variable definitions #
############################
DATASET_SRID="${1}"
DATASET_CITY="${2}"
PG_DB_USER="${3}"
PG_DB_PASSWORD="${4}"
UPDATE_DATA=true
WORKING_DIR="${CURRENT_DIR}"
source "${SCRIPT_DIR}/importData.sh" "${PG_DB_USER}" "${PG_DB_PASSWORD}" "false" "false"
#####################
# Dataset creation #
#####################
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Creating datasets ($(date +%H:%M:%S)):"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
START=$(date +%s)
mkdir -p "${DATA_DIR}"
mkdir -p "${DOWNLOAD_DIR}"
mkdir -p "${WORKING_DIR}"
cd "${WORKING_DIR}"
# Delete old log files
find . -type f -name "*.log" -delete
# Create datasets
CITY_ARR=(${DATASET_CITY//,/ })
SRID_ARR=(${DATASET_SRID//,/ })
for CURRENT_CITY in "${CITY_ARR[@]}"; do
for CURRENT_SRID in "${SRID_ARR[@]}"; do
echo "Creating dataset for city \"${CURRENT_CITY}\" (SRID: ${CURRENT_SRID})"
fn_import_dataset_postgis "${CURRENT_CITY}" "${CURRENT_SRID}"
done
done
END=$(date +%s)
TOTAL=$(( $END - $START ))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Datasets created in directory '${DATA_DIR}'"
echo " Datasets with are still present in the database (only with the last srid specified)"
printf ' Time to create the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment