From 45b4b6e56b7353e467fa4a02080546cf4fded2a6 Mon Sep 17 00:00:00 2001 From: Nikolaus Krismer <nikolaus.krismer@uibk.ac.at> Date: Sat, 13 Feb 2016 14:56:26 +0100 Subject: [PATCH] moved database creation script to isochrone-datamodel --- CHANGELOG.md | 1 + bootstrap/config.sh | 41 +++++++++----- bootstrap/createDatabase.sh | 18 +++--- bootstrap/importData.sh | 26 ++++----- etc/.gitignore | 1 - etc/createDataset.sh | 110 ------------------------------------ 6 files changed, 50 insertions(+), 147 deletions(-) delete mode 100644 etc/.gitignore delete mode 100755 etc/createDataset.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index de2b158..035c554 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ Upcoming version: ----------------- + - moved database creation script to isochrone-datamodel (Nikolaus Krismer) - removing remote cache for geoserver-shell (Nikolaus Krismer) - improved coloring; improved exception handling; downloading java from OTN now (Nikolaus Krismer) - changes due to rename in isochrone-datamodel (Nikolaus Krismer) diff --git a/bootstrap/config.sh b/bootstrap/config.sh index 429a6eb..1b8db42 100755 --- a/bootstrap/config.sh +++ b/bootstrap/config.sh @@ -25,6 +25,8 @@ CACHE_DIR_REMOTE="https://dbis-owncloud.uibk.ac.at/index.php/s/kgjm3CItQJ6P374/d DEPLOY_DIR="/opt" DATA_DIR="$DEPLOY_DIR/data" SETUP_DIR="/setup" +USE_COLOR=true +USE_TIMINGS=false VERBOSE=false WORKING_DIR="/var/log/vagrant_bootstrap" @@ -43,8 +45,10 @@ TOMCAT_PASSWORD="@tomcat_password@" # Do not change anything below this line! -EXEC_POSTGRES="psql" -EXEC_WGET="wget" +EXEC_JAVA=java +EXEC_PGDUMP=pg_dump +EXEC_PSQL=psql +EXEC_WGET=wget POSTGRES_SHARE="$(dirname /usr/pgsql*/.)""/share" if [ ! -d ${POSTGRES_SHARE} ]; then POSTGRES_SHARE="/usr/share/pgsql" @@ -56,7 +60,7 @@ AVAILABLE_ISOCHRONE_DATAMODEL=$([ -d "${DEPLOY_DIR}/isochrone-datamodel" ] && ec AVAILABLE_ISOCHRONE_TOOLS=$([ -f "${DEPLOY_DIR}/isochrone-tools.jar" ] && echo true || echo false) AVAILABLE_NEO4J=$([ -d "${DEPLOY_DIR}/neo4j" ] && echo true || echo false) AVAILABLE_NEO4J_SPATIAL=$([ -d "${DEPLOY_DIR}/neo4j-spatial" ] && echo true || echo false) -AVAILABLE_POSTGRES=$(type ${EXEC_POSTGRES} >/dev/null 2>&1 && echo true || echo false) +AVAILABLE_POSTGRES=$(type ${EXEC_PSQL} >/dev/null 2>&1 && echo true || echo false) AVAILABLE_POSTGIS=$([ -f "${POSTGRES_SHARE}/extension/postgis.control" ] && echo true || echo false) AVAILABLE_SPATIALITE_ADMIN=$([ -f "${DEPLOY_DIR}/phpliteadmin/phpliteadmin.config.php" ] && echo true || echo false) @@ -151,18 +155,27 @@ function fn_echo() { local COLOR="\e[93m" # default color is light green local DEFAULT_COLOR="\e[39m" - case "${LEVEL,,}" in - "debug") COLOR="\e[34m" ;; # Blue - "error") COLOR="\e[91m" ;; # Light red - "warn") COLOR="\e[93m" ;; # light yellow - *) COLOR="\e[92m" ;; # light green (on level=info, no level set or invalid level given) - esac - - if [ "${LEVEL,,}" = "debug" ] && ! ${VERBOSE}; then - return 0 + if ${USE_TIMINGS}; then + TIME=$(date +"%T.%N") + MESSAGE="[${TIME}] ${MESSAGE}" + fi + + if ${USE_COLOR} ; then + case "${LEVEL,,}" in + "debug") COLOR="\e[34m" ;; # Blue + "error") COLOR="\e[91m" ;; # Light red + "warn") COLOR="\e[93m" ;; # light yellow + *) COLOR="\e[92m" ;; # light green (on level=info, no level set or invalid level given) + esac + + if [ "${LEVEL,,}" = "debug" ] && ! ${VERBOSE}; then + return 0 + fi + + echo -e "${COLOR}${MESSAGE}${DEFAULT_COLOR}" + else + echo "${MESSAGE}" fi - - echo -e "${COLOR}${MESSAGE}${DEFAULT_COLOR}" } function fn_service_start() { diff --git a/bootstrap/createDatabase.sh b/bootstrap/createDatabase.sh index 8c212a2..e5fadc6 100755 --- a/bootstrap/createDatabase.sh +++ b/bootstrap/createDatabase.sh @@ -43,19 +43,19 @@ cd $WORKING_DIR if $AVAILABLE_POSTGRES; then fn_echo "Creating postgresql database \"$PG_DB_NAME\"" - sudo -u "postgres" ${EXEC_POSTGRES} -d "template1" -c "CREATE DATABASE \"$PG_DB_NAME\";" >> $WORKING_DIR/setup_database.log 2>&1 - sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "CREATE USER \"$PG_DB_USER\" WITH PASSWORD '$PG_DB_PASSWORD';" >> $WORKING_DIR/setup_database.log 2>&1 - sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "ALTER DATABASE \"$PG_DB_NAME\" OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1 - sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "GRANT ALL PRIVILEGES ON DATABASE \"$PG_DB_NAME\" TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1 - sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "ALTER SCHEMA public OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1 - sudo -u "postgres" ${EXEC_POSTGRES} -d "$PG_DB_NAME" -c "ALTER ROLE \"$PG_DB_USER\" SUPERUSER;" >> $WORKING_DIR/setup_database.log 2>&1 + sudo -u "postgres" ${EXEC_PSQL} -d "template1" -c "CREATE DATABASE \"$PG_DB_NAME\";" >> $WORKING_DIR/setup_database.log 2>&1 + sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "CREATE USER \"$PG_DB_USER\" WITH PASSWORD '$PG_DB_PASSWORD';" >> $WORKING_DIR/setup_database.log 2>&1 + sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "ALTER DATABASE \"$PG_DB_NAME\" OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1 + sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "GRANT ALL PRIVILEGES ON DATABASE \"$PG_DB_NAME\" TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1 + sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "ALTER SCHEMA public OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1 + sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "ALTER ROLE \"$PG_DB_USER\" SUPERUSER;" >> $WORKING_DIR/setup_database.log 2>&1 - PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_POSTGRES} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/pg_procedures.sql >> $WORKING_DIR/setup_database.log 2>&1 + PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/pg_procedures.sql >> $WORKING_DIR/setup_database.log 2>&1 if $AVAILABLE_POSTGIS; then fn_echo "Registering postgis extension for database \"$PG_DB_NAME\"" - PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_POSTGRES} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -c "CREATE EXTENSION postgis;" >> $WORKING_DIR/setup_database.log 2>&1 - PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_POSTGRES} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/srid_82344_insert.sql >> $WORKING_DIR/setup_database.log 2>&1 + PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -c "CREATE EXTENSION postgis;" >> $WORKING_DIR/setup_database.log 2>&1 + PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/srid_82344_insert.sql >> $WORKING_DIR/setup_database.log 2>&1 fi if $AVAILABLE_ISOCHRONE_DATAMODEL; then diff --git a/bootstrap/importData.sh b/bootstrap/importData.sh index 70a8da1..a49244f 100755 --- a/bootstrap/importData.sh +++ b/bootstrap/importData.sh @@ -59,24 +59,24 @@ function fn_create_synthetic_networks() { fn_echo "Importing data for synthetic networks" # GridNetwork - java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1 + ${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1 if [ $? -ne 0 ]; then fn_echo "An error occurred while creating the grid network. The process will exit now with status $?" "ERROR" exit $? fi - java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d "$DENSITY" >> "$WORKING_DIR/generate_gridNetwork.log" 2>&1 + ${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d "$DENSITY" >> "$WORKING_DIR/generate_gridNetwork.log" 2>&1 if [ $? -ne 0 ]; then fn_echo "An error occurred while creating the node_density table for 'grid_s100' using isochrone-tools. The process will exit now with status $?" "ERROR" exit $? fi # SpiderNetwork - java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1 + ${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1 if [ $? -ne 0 ]; then fn_echo "An error occurred while creating the spider network. The process will exit now with status $?" "ERROR" exit $? fi - java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d "$DENSITY" >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1 + ${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d "$DENSITY" >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1 if [ $? -ne 0 ]; then fn_echo "An error occurred while creating the node_density table for 'spider_l1000' using isochrone-tools. The process will exit now with status $?" "ERROR" exit $? @@ -107,7 +107,7 @@ function fn_import_dataset_neo4j_spatial() { CITY=${CITY,,} # We copy data from postgis for specified dataset (after postgis data import) - java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.exporter.ImportDataNeo4j -t "${CITY}" > "$WORKING_DIR/import_neo4j_$CITY.log" 2>&1 + ${EXEC_JAVA} -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.exporter.ImportDataNeo4j -t "${CITY}" > "$WORKING_DIR/import_neo4j_$CITY.log" 2>&1 if [ $? -ne 0 ]; then fn_echo "An error occurred while importing data into neo4j using isochrone-tools. The process will exit now with status $?" "ERROR" exit $? @@ -128,14 +128,14 @@ function fn_import_dataset_postgis() { cp $DOWNLOAD_DIR/$SQL_EXPORT_FILE $DATA_DIR/$SQL_EXPORT_FILE >> /dev/null 2>&1 fn_echo " - importing SQL export into PostGIS database" - gunzip -c $DATA_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost "$PG_DB_NAME" >> "$WORKING_DIR/import_datamodel_$CITY_$SRID.log" 2>&1 + gunzip -c $DATA_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost "$PG_DB_NAME" >> "$WORKING_DIR/import_datamodel_$CITY_$SRID.log" 2>&1 fn_echo " - getting table permissions" - tables=`PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$PG_DB_NAME"` + tables=`PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$PG_DB_NAME"` fn_echo " - fixing table permissions" for tbl in $tables; do - PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -c "ALTER TABLE $tbl OWNER TO $PG_DB_USER" "$PG_DB_NAME" + PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -c "ALTER TABLE $tbl OWNER TO $PG_DB_USER" "$PG_DB_NAME" done fi @@ -154,13 +154,13 @@ function fn_import_dataset_postgis() { fi fn_echo " - copying tables to isochrone database" - # If working with multiple TARGET_SRIDs we have to delete an eventually existing nodes_density table here, so on import later a DROP TABLE ${CITY}_nodes works wihtout CASCADE (pg_dump does not use CASCADE) - PGPASSWORD="$PG_DB_PASSWORD" psql -qAt -U "$PG_DB_USER" -h localhost -d isochrone -c "DROP TABLE IF EXISTS ${CITY}_nodes_density" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 - PGPASSWORD="spatial" pg_dump -U spatial -h localhost -d spatial --clean --if-exists --no-privileges --no-owner -t "transformed.${CITY}_*" | sed -e "s/transformed/public/g" | PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost -d isochrone >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 + # If working with multiple TARGET_SRIDs we have to delete an eventually existing nodes_density table here, so on import later a DROP TABLE ${CITY}_nodes works wihtout CASCADE (${EXEC_PGDUMP} does not use CASCADE) + PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -d isochrone -c "DROP TABLE IF EXISTS ${CITY}_nodes_density" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 + PGPASSWORD="spatial" ${EXEC_PGDUMP} -U spatial -h localhost -d spatial --clean --if-exists --no-privileges --no-owner -t "transformed.${CITY}_*" | sed -e "s/transformed/public/g" | PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -d isochrone >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 fi if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then - java -cp "${DEPLOY_DIR}/isochrone-tools.jar" "at.uibk.dbis.isochrone.generator.density.DensityGenerator" -t "${CITY}" -d "${DENSITY}" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 + ${EXEC_JAVA} -cp "${DEPLOY_DIR}/isochrone-tools.jar" "at.uibk.dbis.isochrone.generator.density.DensityGenerator" -t "${CITY}" -d "${DENSITY}" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 if [ $? -ne 0 ]; then fn_echo "An error occurred while creating the node_density table for '${CITY}' using isochrone-tools. The process will exit now with status $?" "ERROR" exit $? @@ -170,7 +170,7 @@ function fn_import_dataset_postgis() { if $AVAILABLE_ISOCHRONE_DATAMODEL; then # Exporting resulting database to $DATA_DIR fn_echo " - exporting database dump" - PGPASSWORD="$PG_DB_PASSWORD" pg_dump -U "$PG_DB_USER" -h localhost -d isochrone --clean --compress=5 --if-exists -t "${CITY}_*" --file="$DATA_DIR/$SQL_EXPORT_FILE" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 + PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PGDUMP} -U "$PG_DB_USER" -h localhost -d isochrone --clean --compress=5 --if-exists -t "${CITY}_*" --file="$DATA_DIR/$SQL_EXPORT_FILE" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 fi } diff --git a/etc/.gitignore b/etc/.gitignore deleted file mode 100644 index bf0824e..0000000 --- a/etc/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.log \ No newline at end of file diff --git a/etc/createDataset.sh b/etc/createDataset.sh deleted file mode 100755 index 20a5b1c..0000000 --- a/etc/createDataset.sh +++ /dev/null @@ -1,110 +0,0 @@ -#! /bin/bash -############################# -# This script creates datasets (e.g. from the road network of bolzano) -# from data that is downloaded from osm (OpenStreetMap) and then imported -# into the postgis database. -# -# In difference to importData.sh this only creates the given datasets with the -# given SRIDs by impoering them into postgis. -# -# Note that you need some synlinks in directoy /opt/ (to isochrone-datamodel -# and isochrone-tools project) for this script to work correctly. -############################# - -if [ "$EUID" -ne "0" ]; then - echo '- This script must be run as root!' - exit 1 -fi - -######################################################################## -# Config import (so we can override some things like working directory # -######################################################################## - -SCRIPT="$(readlink -f ${BASH_SOURCE[0]})" -CURRENT_DIR="$(dirname ${SCRIPT})" -SCRIPT_DIR="${CURRENT_DIR}/../bootstrap" -if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then - # If config.properties is not besides the bash-script (most likely because vagrant uploaded it into the guest) - # we will try to find it in the shared folder - SCRIPT_DIR="/vagrant/bootstrap" -fi -if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then - echo '- No variable declarations found (config.properties file not found)!' - exit 1; -fi -source "${SCRIPT_DIR}/config.sh" - -############################ -# Precondition checks # -############################ - - -if ! $AVAILABLE_ISOCHRONE_DATAMODEL; then - echo " - Project folder isochrone-datamodel not present in directory '${DEPLOY_DIR}'" - exit 1 -fi - -if ! $AVAILABLE_ISOCHRONE_TOOLS; then - echo " - Project archive isochrone-tools not present in directory '${DEPLOY_DIR}'" - exit 1 -fi - -if ! $AVAILABLE_POSTGRES; then - echo " - PostgreSQL database not installed on this system" - exit 1 -fi - -if ! $AVAILABLE_POSTGIS; then - echo " - Postgis extension not installed on this system" - exit 1 -fi - -############################ -# Variable definitions # -############################ - -DATASET_SRID="${1}" -DATASET_CITY="${2}" -PG_DB_USER="${3}" -PG_DB_PASSWORD="${4}" -UPDATE_DATA=true -WORKING_DIR="${CURRENT_DIR}" - -source "${SCRIPT_DIR}/importData.sh" "${PG_DB_USER}" "${PG_DB_PASSWORD}" "false" "false" - -##################### -# Dataset creation # -##################### - -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo " Creating datasets ($(date +%H:%M:%S)):" -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - -START=$(date +%s) - -mkdir -p "${DATA_DIR}" -mkdir -p "${DOWNLOAD_DIR}" -mkdir -p "${WORKING_DIR}" -cd "${WORKING_DIR}" - -# Delete old log files -find . -type f -name "*.log" -delete - -# Create datasets -CITY_ARR=(${DATASET_CITY//,/ }) -SRID_ARR=(${DATASET_SRID//,/ }) -for CURRENT_CITY in "${CITY_ARR[@]}"; do - for CURRENT_SRID in "${SRID_ARR[@]}"; do - echo "Creating dataset for city \"${CURRENT_CITY}\" (SRID: ${CURRENT_SRID})" - fn_import_dataset_postgis "${CURRENT_CITY}" "${CURRENT_SRID}" - done -done - -END=$(date +%s) -TOTAL=$(( $END - $START )) - -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -echo " Datasets created in directory '${DATA_DIR}'" -echo " Datasets with are still present in the database (only with the last srid specified)" -printf ' Time to create the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60)) -echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -- GitLab