Skip to content
Snippets Groups Projects
Commit f1105442 authored by User expired's avatar User expired
Browse files

upgrading to latest postgresql and postgis version

parent b277666e
No related branches found
No related tags found
No related merge requests found
Pipeline #
Hide whitespace changes
Inline Side-by-side
CHANGELOG.md
+ 1
− 0
View file @ f1105442
Upcoming version: Upcoming version:
----------------- -----------------
- upgrading to latest postgresql and postgis version (Nikolaus Krismer)
- updating dropTables to use CASCADE (Nikolaus Krismer) - updating dropTables to use CASCADE (Nikolaus Krismer)
- added support for 3d datamodels (Nikolaus Krismer) - added support for 3d datamodels (Nikolaus Krismer)
- changing parameter usage for osmPti2mmds (Nikolaus Krismer) - changing parameter usage for osmPti2mmds (Nikolaus Krismer)
......
Vagrantfile
+ 2
− 1
View file @ f1105442
...@@ -87,12 +87,13 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| ...@@ -87,12 +87,13 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
#config.vm.provision "shell", path: "bootstrap/activateDatabaseLogging.sh" #config.vm.provision "shell", path: "bootstrap/activateDatabaseLogging.sh"
# Network configuration # Network configuration
config.vm.network :forwarded_port, guest: 5050, host: 5050, id: "pgadmin4", auto_correct: false
config.vm.network :forwarded_port, guest: 5432, host: 5432, id: "postgresql", auto_correct: false config.vm.network :forwarded_port, guest: 5432, host: 5432, id: "postgresql", auto_correct: false
config.vm.network :forwarded_port, guest: 7473, host: 7473, id: "neo4j-ssl", auto_correct: false config.vm.network :forwarded_port, guest: 7473, host: 7473, id: "neo4j-ssl", auto_correct: false
config.vm.network :forwarded_port, guest: 7474, host: 7474, id: "neo4j", auto_correct: false config.vm.network :forwarded_port, guest: 7474, host: 7474, id: "neo4j", auto_correct: false
config.vm.network :forwarded_port, guest: 80, host: 8000, id: "httpd", auto_correct: false config.vm.network :forwarded_port, guest: 80, host: 8000, id: "httpd", auto_correct: false
unless (isLocalDeploy || isJenkinsDeploy) unless (isLocalDeploy || isJenkinsDeploy)
config.vm.network :forwarded_port, guest: 8080, host: 8080, id: "tomcat", auto_correct: false config.vm.network :forwarded_port, guest: 8080, host: 8080, id: "tomcat", auto_correct: false
end end
# config.vm.network :private_network, :ip => '192.168.4.50' # config.vm.network :private_network, :ip => '192.168.4.50'
# config.vm.network :public_network # config.vm.network :public_network
......
bootstrap/config.sh
+ 9
− 5
View file @ f1105442
...@@ -16,10 +16,9 @@ IMPORT_DATA_SPATIALITE=true ...@@ -16,10 +16,9 @@ IMPORT_DATA_SPATIALITE=true
# Comma separated list of srids for which datasets will be generated # Comma separated list of srids for which datasets will be generated
# This is useful when creating multiple datasets (when creating/updating data from scratch) # This is useful when creating multiple datasets (when creating/updating data from scratch)
# Note that only the last SRID in the list will be kept in the database, all the others will be overwritten # Note that only the last SRID in the list will be kept in the database, all the others will be overwritten
#TARGET_SRID="3857,4326" #TARGET_SRID="3857,geography,4326"
TARGET_SRID="4326" TARGET_SRID="4326"
UPDATE_DATA=false UPDATE_DATA=false
USE_3D=true
CACHE_DIR_LOCAL="/tmp/vagrant-cache" CACHE_DIR_LOCAL="/tmp/vagrant-cache"
CACHE_DIR_REMOTE="https://dbis-owncloud.uibk.ac.at/index.php/s/kgjm3CItQJ6P374/download?path=%2F2016&files=" CACHE_DIR_REMOTE="https://dbis-owncloud.uibk.ac.at/index.php/s/kgjm3CItQJ6P374/download?path=%2F2016&files="
...@@ -32,12 +31,17 @@ VERBOSE=false ...@@ -32,12 +31,17 @@ VERBOSE=false
WORKING_DIR="/var/log/vagrant_bootstrap" WORKING_DIR="/var/log/vagrant_bootstrap"
# Parameters that can also (some must) be passed as arguments # Parameters that can also (some must) be passed as arguments
DB_NAME="isochrone"
DB_USERNAME="@db_username@"
DB_PASSWORD="@db_password@"
DEPLOY_ALL_DATASETS=false DEPLOY_ALL_DATASETS=false
IS_JENKINS_DEPLOY=false IS_JENKINS_DEPLOY=false
IS_LOCAL_DEPLOY=false IS_LOCAL_DEPLOY=false
PG_DB_NAME="isochrone" # leave SRTM_RESOLUTION empty for 2D dataset creation
PG_DB_USER="@db_username@" # provide username and passwort when using SRTM_Resolution=1 with UPDATE_DATA=true
PG_DB_PASSWORD="@db_password@" SRTM_RESOLUTION=3
SRTM_USERNAME=""
SRTM_PASSWORD=""
TOMCAT_PASSWORD="@tomcat_password@" TOMCAT_PASSWORD="@tomcat_password@"
############################ ############################
......
bootstrap/createDatabase.sh
+ 17
− 17
View file @ f1105442
...@@ -25,8 +25,8 @@ if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then ...@@ -25,8 +25,8 @@ if [ ! -f "${SCRIPT_DIR}/config.sh" ]; then
fi fi
source "${SCRIPT_DIR}/config.sh" source "${SCRIPT_DIR}/config.sh"
fn_arg2string PG_DB_USER "${1}" "${PG_DB_USER}" fn_arg2string DB_USERNAME "${1}" "${DB_USERNAME}"
fn_arg2string PG_DB_PASSWORD "${2}" "${PG_DB_PASSWORD}" fn_arg2string DB_PASSWORD "${2}" "${DB_PASSWORD}"
###################### ######################
# Database creation # # Database creation #
...@@ -43,32 +43,32 @@ cd "$WORKING_DIR" ...@@ -43,32 +43,32 @@ cd "$WORKING_DIR"
touch "$WORKING_DIR/database_creation.lock" touch "$WORKING_DIR/database_creation.lock"
if $AVAILABLE_POSTGRES; then if $AVAILABLE_POSTGRES; then
fn_echo "Creating postgresql database \"$PG_DB_NAME\"" fn_echo "Creating postgresql database \"$DB_NAME\""
sudo -u "postgres" ${EXEC_PSQL} -d "template1" -c "CREATE DATABASE \"$PG_DB_NAME\";" >> $WORKING_DIR/setup_database.log 2>&1 sudo -u "postgres" ${EXEC_PSQL} -d "template1" -c "CREATE DATABASE \"$DB_NAME\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "CREATE USER \"$PG_DB_USER\" WITH PASSWORD '$PG_DB_PASSWORD';" >> $WORKING_DIR/setup_database.log 2>&1 sudo -u "postgres" ${EXEC_PSQL} -d "$DB_NAME" -c "CREATE USER \"$DB_USERNAME\" WITH PASSWORD '$DB_PASSWORD';" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "ALTER DATABASE \"$PG_DB_NAME\" OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1 sudo -u "postgres" ${EXEC_PSQL} -d "$DB_NAME" -c "ALTER DATABASE \"$DB_NAME\" OWNER TO \"$DB_USERNAME\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "GRANT ALL PRIVILEGES ON DATABASE \"$PG_DB_NAME\" TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1 sudo -u "postgres" ${EXEC_PSQL} -d "$DB_NAME" -c "GRANT ALL PRIVILEGES ON DATABASE \"$DB_NAME\" TO \"$DB_USERNAME\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "ALTER SCHEMA public OWNER TO \"$PG_DB_USER\";" >> $WORKING_DIR/setup_database.log 2>&1 sudo -u "postgres" ${EXEC_PSQL} -d "$DB_NAME" -c "ALTER SCHEMA public OWNER TO \"$DB_USERNAME\";" >> $WORKING_DIR/setup_database.log 2>&1
sudo -u "postgres" ${EXEC_PSQL} -d "$PG_DB_NAME" -c "ALTER ROLE \"$PG_DB_USER\" SUPERUSER;" >> $WORKING_DIR/setup_database.log 2>&1 sudo -u "postgres" ${EXEC_PSQL} -d "$DB_NAME" -c "ALTER ROLE \"$DB_USERNAME\" SUPERUSER;" >> $WORKING_DIR/setup_database.log 2>&1
fn_echo "Adding database functions/procedures" fn_echo "Adding database functions/procedures"
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/pg_procedures.sql >> $WORKING_DIR/setup_database.log 2>&1 PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -U "$DB_USERNAME" -h localhost -d "$DB_NAME" -f $SHARED_CONF_DIR/pg_procedures.sql >> $WORKING_DIR/setup_database.log 2>&1
if $AVAILABLE_POSTGIS; then if $AVAILABLE_POSTGIS; then
fn_echo "Registering postgis extension for database \"$PG_DB_NAME\"" fn_echo "Registering postgis extension for database \"$DB_NAME\""
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -c "CREATE EXTENSION postgis;" >> $WORKING_DIR/setup_database.log 2>&1 PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -U "$DB_USERNAME" -h localhost -d "$DB_NAME" -c "CREATE EXTENSION postgis;" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/srid_82344_insert.sql >> $WORKING_DIR/setup_database.log 2>&1 PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -U "$DB_USERNAME" -h localhost -d "$DB_NAME" -f $SHARED_CONF_DIR/srid_82344_insert.sql >> $WORKING_DIR/setup_database.log 2>&1
fn_echo "Adding postgis functions/procedures" fn_echo "Adding postgis functions/procedures"
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost -d "$PG_DB_NAME" -f $SHARED_CONF_DIR/pg_procedures_postgis.sql >> $WORKING_DIR/setup_database.log 2>&1 PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -U "$DB_USERNAME" -h localhost -d "$DB_NAME" -f $SHARED_CONF_DIR/pg_procedures_postgis.sql >> $WORKING_DIR/setup_database.log 2>&1
fi fi
if $AVAILABLE_ISOCHRONE_DATAMODEL; then if $AVAILABLE_ISOCHRONE_DATAMODEL; then
fn_echo "Initializing database schema for isochrone-datamodel (used for testing and datamodel creation)" fn_echo "Initializing database schema for isochrone-datamodel (used for testing and datamodel creation)"
PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost -d "template1" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/db_create.sql" >> $WORKING_DIR/setup_database.log 2>&1 PGPASSWORD="$DB_PASSWORD" psql -U "$DB_USERNAME" -h localhost -d "template1" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/db_create.sql" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost -d "spatial" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/schema_drop.sql" >> $WORKING_DIR/setup_database.log 2>&1 PGPASSWORD="$DB_PASSWORD" psql -U "$DB_USERNAME" -h localhost -d "spatial" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/schema_drop.sql" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="spatial" psql -U "spatial" -h localhost -d "spatial" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/schema_create.sql" >> $WORKING_DIR/setup_database.log 2>&1 PGPASSWORD="spatial" psql -U "spatial" -h localhost -d "spatial" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/schema_create.sql" >> $WORKING_DIR/setup_database.log 2>&1
PGPASSWORD="$PG_DB_PASSWORD" psql -U "$PG_DB_USER" -h localhost -d "spatial" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/extension_create.sql" >> $WORKING_DIR/setup_database.log 2>&1 PGPASSWORD="$DB_PASSWORD" psql -U "$DB_USERNAME" -h localhost -d "spatial" -f "${DEPLOY_DIR}/isochrone-datamodel/at/uibk/dbis/isochrone/datamodel/db/extension_create.sql" >> $WORKING_DIR/setup_database.log 2>&1
fi fi
fi fi
......
bootstrap/importData.sh
+ 20
− 14
View file @ f1105442
...@@ -31,10 +31,12 @@ if ! $IMPORT_DATA_POSTGIS; then ...@@ -31,10 +31,12 @@ if ! $IMPORT_DATA_POSTGIS; then
exit 0 exit 0
fi fi
fn_arg2string PG_DB_USER "${1}" "${PG_DB_USER}" fn_arg2string DB_USERNAME "${1}" "${DB_USERNAME}"
fn_arg2string PG_DB_PASSWORD "${2}" "${PG_DB_PASSWORD}" fn_arg2string DB_PASSWORD "${2}" "${DB_PASSWORD}"
fn_arg2boolean DEPLOY_ALL_DATASETS "${3}" "${DEPLOY_ALL_DATASETS}" fn_arg2boolean DEPLOY_ALL_DATASETS "${3}" "${DEPLOY_ALL_DATASETS}"
fn_arg2boolean DEPLOY_ANY_DATASET "${4}" "true" fn_arg2boolean DEPLOY_ANY_DATASET "${4}" "true"
fn_arg2string SRTM_USERNAME "${5}" "${SRTM_USERNAME}"
fn_arg2string SRTM_PASSWORD "${6}" "${SRTM_PASSWORD}"
###################################### ######################################
# Configuration (tool versions, ...) # # Configuration (tool versions, ...) #
...@@ -42,6 +44,10 @@ fn_arg2boolean DEPLOY_ANY_DATASET "${4}" "true" ...@@ -42,6 +44,10 @@ fn_arg2boolean DEPLOY_ANY_DATASET "${4}" "true"
#DENSITY="60,120,180,240,300" #DENSITY="60,120,180,240,300"
DENSITY="100,200,300,400,500,600,700,800,900,1000" DENSITY="100,200,300,400,500,600,700,800,900,1000"
USE_3D=false
if [ -n $SRTM_RESOLUTION ]; then
USE_3D=true
fi
############################ ############################
# Function definitions # # Function definitions #
...@@ -115,7 +121,7 @@ function fn_import_dataset_postgis() { ...@@ -115,7 +121,7 @@ function fn_import_dataset_postgis() {
local SQL_EXPORT_FILE="${NAME,,}_export_${SRID}.sql.gz" local SQL_EXPORT_FILE="${NAME,,}_export_${SRID}.sql.gz"
if $USE_3D; then if $USE_3D; then
SQL_EXPORT_FILE="${NAME,,}_export_${SRID}_3d.sql.gz" SQL_EXPORT_FILE="${NAME,,}_export_${SRID}_srtm${SRTM_RESOLUTION}.sql.gz"
fi fi
if ! $UPDATE_DATA; then if ! $UPDATE_DATA; then
...@@ -124,14 +130,14 @@ function fn_import_dataset_postgis() { ...@@ -124,14 +130,14 @@ function fn_import_dataset_postgis() {
cp $DOWNLOAD_DIR/$SQL_EXPORT_FILE $DATA_DIR/$SQL_EXPORT_FILE >> /dev/null 2>&1 cp $DOWNLOAD_DIR/$SQL_EXPORT_FILE $DATA_DIR/$SQL_EXPORT_FILE >> /dev/null 2>&1
fn_echo " - importing SQL export into PostGIS database" fn_echo " - importing SQL export into PostGIS database"
gunzip -c $DATA_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -U "$PG_DB_USER" -h localhost "$PG_DB_NAME" >> "$WORKING_DIR/import_datamodel_$CITY_$SRID.log" 2>&1 gunzip -c $DATA_DIR/$SQL_EXPORT_FILE | PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -U "$DB_USERNAME" -h localhost "$DB_NAME" >> "$WORKING_DIR/import_datamodel_$CITY_$SRID.log" 2>&1
fn_echo " - getting table permissions" fn_echo " - getting table permissions"
tables=`PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$PG_DB_NAME"` tables=`PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$DB_USERNAME" -h localhost -c "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tableowner = 'postgres';" "$DB_NAME"`
fn_echo " - fixing table permissions" fn_echo " - fixing table permissions"
for tbl in $tables; do for tbl in $tables; do
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -c "ALTER TABLE $tbl OWNER TO $PG_DB_USER" "$PG_DB_NAME" PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$DB_USERNAME" -h localhost -c "ALTER TABLE $tbl OWNER TO $DB_USERNAME" "$DB_NAME"
done done
fi fi
...@@ -143,7 +149,7 @@ function fn_import_dataset_postgis() { ...@@ -143,7 +149,7 @@ function fn_import_dataset_postgis() {
if $AVAILABLE_ISOCHRONE_DATAMODEL && [ ! -f "$DATA_DIR/$SQL_EXPORT_FILE" ]; then if $AVAILABLE_ISOCHRONE_DATAMODEL && [ ! -f "$DATA_DIR/$SQL_EXPORT_FILE" ]; then
fn_echo " - creating datamodel using isochrone-datamodel" fn_echo " - creating datamodel using isochrone-datamodel"
# Create datamodel using isochrone-datamodel project # Create datamodel using isochrone-datamodel project
DB_USERNAME="$PG_DB_USER" DB_PASSWORD="$PG_DB_PASSWORD" "$DEPLOY_DIR/isochrone-datamodel/osmPti2mmds.sh" -d -s -b -l -t ${SRID} -c "${CITY}" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 DB_USERNAME="$DB_USERNAME" DB_PASSWORD="$DB_PASSWORD" "$DEPLOY_DIR/isochrone-datamodel/osmPti2mmds.sh" -d -s -b -l -t ${SRID} -c "${CITY}" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
fn_echo "An error occurred while creating datamodel using isochrone-datamodel. The process will exit now with status $?" "ERROR" fn_echo "An error occurred while creating datamodel using isochrone-datamodel. The process will exit now with status $?" "ERROR"
exit $? exit $?
...@@ -151,22 +157,22 @@ function fn_import_dataset_postgis() { ...@@ -151,22 +157,22 @@ function fn_import_dataset_postgis() {
fn_echo " - copying tables to isochrone database" fn_echo " - copying tables to isochrone database"
# If working with multiple TARGET_SRIDs we have to delete an eventually existing nodes_density table here, so on import later a DROP TABLE ${CITY}_nodes works wihtout CASCADE (${EXEC_PGDUMP} does not use CASCADE) # If working with multiple TARGET_SRIDs we have to delete an eventually existing nodes_density table here, so on import later a DROP TABLE ${CITY}_nodes works wihtout CASCADE (${EXEC_PGDUMP} does not use CASCADE)
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -d isochrone -c "DROP TABLE IF EXISTS ${CITY}_nodes_density" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$DB_USERNAME" -h localhost -d isochrone -c "DROP TABLE IF EXISTS ${CITY}_nodes_density" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
PGPASSWORD="spatial" ${EXEC_PGDUMP} -U spatial -h localhost -d spatial --clean --if-exists --no-privileges --no-owner -t "transformed.${CITY}_*" | sed -e "s/transformed/public/g" | PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -d isochrone >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 PGPASSWORD="spatial" ${EXEC_PGDUMP} -U spatial -h localhost -d spatial --clean --if-exists --no-privileges --no-owner -t "transformed.${CITY}_*" | sed -e "s/transformed/public/g" | PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$DB_USERNAME" -h localhost -d isochrone >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
if $USE_3D; then if $USE_3D; then
fn_echo " - adding node elevation for city \"${CITY}\" (using isochrone-tools)" fn_echo " - adding node elevation for city \"${CITY}\" (using isochrone-tools)"
${EXEC_JAVA} -cp "${DEPLOY_DIR}/isochrone-tools.jar" at.uibk.dbis.isochrone.generator.elevation.SRTMGeneratorNode -ds "${CITY}" >> "$WORKING_DIR/create_elevation_${CITY}_${SRID}.log" 2>&1 ${EXEC_JAVA} -cp "${DEPLOY_DIR}/isochrone-tools.jar" at.uibk.dbis.isochrone.generator.elevation.SRTMGeneratorNode -ds "${CITY}" -r ${SRTM_RESOLUTION} -p "${SRTM_PASSWORD}" -u "${SRTM_USERNAME}" >> "$WORKING_DIR/create_elevation_${CITY}_${SRID}.log" 2>&1
fn_check_status fn_check_status
fn_echo " - adding edge elevation for city \"${CITY}\" (using isochrone-tools)" fn_echo " - adding edge elevation for city \"${CITY}\" (using isochrone-tools)"
${EXEC_JAVA} -cp "${DEPLOY_DIR}/isochrone-tools.jar" at.uibk.dbis.isochrone.generator.elevation.SRTMGeneratorEdge -ds "${CITY}" >> "$WORKING_DIR/create_elevation_${CITY}_${SRID}.log" 2>&1 ${EXEC_JAVA} -cp "${DEPLOY_DIR}/isochrone-tools.jar" at.uibk.dbis.isochrone.generator.elevation.SRTMGeneratorEdge -ds "${CITY}" -r ${SRTM_RESOLUTION} -p "${SRTM_PASSWORD}" -u "${SRTM_USERNAME}" >> "$WORKING_DIR/create_elevation_${CITY}_${SRID}.log" 2>&1
fn_check_status fn_check_status
fi fi
fi fi
if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then
local exists_density=$(PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$PG_DB_USER" -h localhost -d isochrone -c "SELECT EXISTS (SELECT * FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid=C.relnamespace) WHERE nspname = 'public' AND relname='${CITY}_nodes_density')") local exists_density=$(PGPASSWORD="$DB_PASSWORD" ${EXEC_PSQL} -qAt -U "$DB_USERNAME" -h localhost -d isochrone -c "SELECT EXISTS (SELECT * FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid=C.relnamespace) WHERE nspname = 'public' AND relname='${CITY}_nodes_density')")
if [ "$exists_density" == "t" ]; then if [ "$exists_density" == "t" ]; then
fn_echo " - nodes_density exists... not re-creating table" fn_echo " - nodes_density exists... not re-creating table"
else else
...@@ -182,7 +188,7 @@ function fn_import_dataset_postgis() { ...@@ -182,7 +188,7 @@ function fn_import_dataset_postgis() {
if $AVAILABLE_ISOCHRONE_DATAMODEL; then if $AVAILABLE_ISOCHRONE_DATAMODEL; then
# Exporting resulting database to $DATA_DIR # Exporting resulting database to $DATA_DIR
fn_echo " - exporting database dump" fn_echo " - exporting database dump"
PGPASSWORD="$PG_DB_PASSWORD" ${EXEC_PGDUMP} -U "$PG_DB_USER" -h localhost -d isochrone --clean --compress=5 --if-exists -t "${CITY}_*" --file="$DATA_DIR/$SQL_EXPORT_FILE" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1 PGPASSWORD="$DB_PASSWORD" ${EXEC_PGDUMP} -U "$DB_USERNAME" -h localhost -d isochrone --clean --compress=5 --if-exists -t "${CITY}_*" --file="$DATA_DIR/$SQL_EXPORT_FILE" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
fi fi
} }
...@@ -210,7 +216,7 @@ function fn_import_spatialite() { ...@@ -210,7 +216,7 @@ function fn_import_spatialite() {
if $AVAILABLE_GDAL && [ ! -f "$DATA_DIR/$SPATIALITE_FILENAME" ]; then if $AVAILABLE_GDAL && [ ! -f "$DATA_DIR/$SPATIALITE_FILENAME" ]; then
fn_echo "Exporting PostGIS data into spatialite database" fn_echo "Exporting PostGIS data into spatialite database"
$DEPLOY_DIR/gdal/apps/ogr2ogr --config PG_LIST_ALL_TABLES YES --config PG_SKIP_VIEWS YES -progress -f "SQLite" "$DATA_DIR/$SPATIALITE_FILENAME" PG:"host=localhost dbname=isochrone user=$PG_DB_USER password=$PG_DB_PASSWORD" -lco LAUNDER=yes -dsco SPATIALITE=yes -lco SPATIAL_INDEX=yes -gt 65536 >> "$WORKING_DIR/import_spatialite.log" 2>&1 $DEPLOY_DIR/gdal/apps/ogr2ogr --config PG_LIST_ALL_TABLES YES --config PG_SKIP_VIEWS YES -progress -f "SQLite" "$DATA_DIR/$SPATIALITE_FILENAME" PG:"host=localhost dbname=isochrone user=$DB_USERNAME password=$DB_PASSWORD" -lco LAUNDER=yes -dsco SPATIALITE=yes -lco SPATIAL_INDEX=yes -gt 65536 >> "$WORKING_DIR/import_spatialite.log" 2>&1
fi fi
if [ -f "$DATA_DIR/$SPATIALITE_FILENAME" ]; then if [ -f "$DATA_DIR/$SPATIALITE_FILENAME" ]; then
......
bootstrap/prepareEnvironment.sh
+ 27
− 18
View file @ f1105442
...@@ -50,7 +50,7 @@ INSTALL_NEO4J=false ...@@ -50,7 +50,7 @@ INSTALL_NEO4J=false
INSTALL_NEO4J_SPATIAL=false INSTALL_NEO4J_SPATIAL=false
INSTALL_POSTGIS=false INSTALL_POSTGIS=false
INSTALL_POSTGRES=false INSTALL_POSTGRES=false
INSTALL_POSTGRES_ADMIN=true INSTALL_PGADMIN=true
INSTALL_PROJECT_PSIPROBE=true INSTALL_PROJECT_PSIPROBE=true
INSTALL_SPATIALITE=false INSTALL_SPATIALITE=false
INSTALL_SPATIALITE_ADMIN=false INSTALL_SPATIALITE_ADMIN=false
...@@ -76,7 +76,7 @@ fn_arg2boolean IS_JENKINS_DEPLOY "${3}" "${IS_JENKINS_DEPLOY}" ...@@ -76,7 +76,7 @@ fn_arg2boolean IS_JENKINS_DEPLOY "${3}" "${IS_JENKINS_DEPLOY}"
# Configuration (tool versions, ...) # # Configuration (tool versions, ...) #
###################################### ######################################
GDAL_VERSION="2.0.0" GDAL_VERSION="2.1.1"
GDAL_FILE="gdal-$GDAL_VERSION.tar.gz" GDAL_FILE="gdal-$GDAL_VERSION.tar.gz"
GDAL_DOWNLOAD_URL="http://download.osgeo.org/gdal/$GDAL_VERSION/$GDAL_FILE" GDAL_DOWNLOAD_URL="http://download.osgeo.org/gdal/$GDAL_VERSION/$GDAL_FILE"
GEOSERVER_VERSION="2.9.1" GEOSERVER_VERSION="2.9.1"
...@@ -128,8 +128,8 @@ POSTGIS_VERSION="auto" # activate to install from repository ...@@ -128,8 +128,8 @@ POSTGIS_VERSION="auto" # activate to install from repository
#POSTGIS_VERSION="2.1.5" # activate to install from source #POSTGIS_VERSION="2.1.5" # activate to install from source
POSTGRESQL_DATA_DIR="/var/lib/pgsql/data" POSTGRESQL_DATA_DIR="/var/lib/pgsql/data"
POSTGRESQL_SERVICE_NAME="postgresql" POSTGRESQL_SERVICE_NAME="postgresql"
POSTGRESQL_SHORT_VERSION="95" POSTGRESQL_SHORT_VERSION="96"
POSTGRESQL_VERSION="9.5" POSTGRESQL_VERSION="9.6"
REPO_ELGIS="none" REPO_ELGIS="none"
REPO_EPEL="none" REPO_EPEL="none"
REPO_POSTGRESQL="none" # will be set later (depending on architecture) REPO_POSTGRESQL="none" # will be set later (depending on architecture)
...@@ -174,13 +174,13 @@ if [ "$DISTRI" == "CentOS" ]; then ...@@ -174,13 +174,13 @@ if [ "$DISTRI" == "CentOS" ]; then
DISTRI_TYPE="CentOS 6" DISTRI_TYPE="CentOS 6"
REPO_ELGIS="http://elgis.argeo.org/repos/6/elgis-release-6-6_0.noarch.rpm" REPO_ELGIS="http://elgis.argeo.org/repos/6/elgis-release-6-6_0.noarch.rpm"
REPO_EPEL="https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm" REPO_EPEL="https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm"
REPO_POSTGRESQL_32="http://yum.postgresql.org/$POSTGRESQL_VERSION/redhat/rhel-6-i386/pgdg-centos$POSTGRESQL_SHORT_VERSION-$POSTGRESQL_VERSION-2.noarch.rpm" REPO_POSTGRESQL_32="http://yum.postgresql.org/$POSTGRESQL_VERSION/redhat/rhel-6-i386/pgdg-centos$POSTGRESQL_SHORT_VERSION-$POSTGRESQL_VERSION-3.noarch.rpm"
REPO_POSTGRESQL_64="http://yum.postgresql.org/$POSTGRESQL_VERSION/redhat/rhel-6-x86_64/pgdg-centos$POSTGRESQL_SHORT_VERSION-$POSTGRESQL_VERSION-2.noarch.rpm" REPO_POSTGRESQL_64="http://yum.postgresql.org/$POSTGRESQL_VERSION/redhat/rhel-6-x86_64/pgdg-centos$POSTGRESQL_SHORT_VERSION-$POSTGRESQL_VERSION-3.noarch.rpm"
DISTRI_SUPPORTED=true DISTRI_SUPPORTED=true
elif printf "$DISTRI_NAME" | egrep -q " 7." ; then elif printf "$DISTRI_NAME" | egrep -q " 7." ; then
DISTRI_TYPE="CentOS 7" DISTRI_TYPE="CentOS 7"
REPO_EPEL="https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" REPO_EPEL="https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm"
REPO_POSTGRESQL_64="http://yum.postgresql.org/$POSTGRESQL_VERSION/redhat/rhel-7-x86_64/pgdg-centos$POSTGRESQL_SHORT_VERSION-$POSTGRESQL_VERSION-2.noarch.rpm" REPO_POSTGRESQL_64="http://yum.postgresql.org/$POSTGRESQL_VERSION/redhat/rhel-7-x86_64/pgdg-centos$POSTGRESQL_SHORT_VERSION-$POSTGRESQL_VERSION-3.noarch.rpm"
DISTRI_SUPPORTED=$IS_64_BIT DISTRI_SUPPORTED=$IS_64_BIT
fi fi
fi fi
...@@ -229,7 +229,7 @@ if $IMPORT_DATA_SPATIALITE || $INSTALL_GDAL || $INSTALL_SPATIALITE_ADMIN; then ...@@ -229,7 +229,7 @@ if $IMPORT_DATA_SPATIALITE || $INSTALL_GDAL || $INSTALL_SPATIALITE_ADMIN; then
INSTALL_SPATIALITE=true INSTALL_SPATIALITE=true
INSTALL_SPATIALITE_ADMIN=true INSTALL_SPATIALITE_ADMIN=true
fi fi
if $INSTALL_POSTGRES_ADMIN || $INSTALL_SPATIALITE_ADMIN; then if $INSTALL_SPATIALITE_ADMIN; then
INSTALL_HTTPD=true INSTALL_HTTPD=true
fi fi
if $INSTALL_GEOSERVER || $INSTALL_ISOCHRONE_LOCAL || $INSTALL_ISOCHRONE_RELEASE || $INSTALL_ISOCHRONE_SNAPSHOT || $INSTALL_PROJECT_PSIPROBE; then if $INSTALL_GEOSERVER || $INSTALL_ISOCHRONE_LOCAL || $INSTALL_ISOCHRONE_RELEASE || $INSTALL_ISOCHRONE_SNAPSHOT || $INSTALL_PROJECT_PSIPROBE; then
...@@ -242,7 +242,7 @@ fi ...@@ -242,7 +242,7 @@ fi
if $INSTALL_ISOCHRONE_TOOLS || $INSTALL_ISOCHRONE_DATAMODEL || $INSTALL_TOMCAT || $INSTALL_GEOSERVER || $INSTALL_NEO4J || $INSTALL_NEO4J_SPATIAL || $IMPORT_DATA_NEO4J_SPATIAL; then if $INSTALL_ISOCHRONE_TOOLS || $INSTALL_ISOCHRONE_DATAMODEL || $INSTALL_TOMCAT || $INSTALL_GEOSERVER || $INSTALL_NEO4J || $INSTALL_NEO4J_SPATIAL || $IMPORT_DATA_NEO4J_SPATIAL; then
INSTALL_JAVA=true INSTALL_JAVA=true
fi fi
if $INSTALL_POSTGRES_ADMIN || $INSTALL_POSTGIS; then if $INSTALL_PGADMIN || $INSTALL_POSTGIS; then
INSTALL_POSTGRES=true INSTALL_POSTGRES=true
fi fi
...@@ -370,7 +370,7 @@ if $INSTALL_SPATIALITE_ADMIN; then fn_echo " - phpLiteAdmin $PHPLITEADMIN_VERSI ...@@ -370,7 +370,7 @@ if $INSTALL_SPATIALITE_ADMIN; then fn_echo " - phpLiteAdmin $PHPLITEADMIN_VERSI
if $INSTALL_GDAL; then fn_echo " - gdal $GDAL_VERSION"; fi if $INSTALL_GDAL; then fn_echo " - gdal $GDAL_VERSION"; fi
if $INSTALL_TOMCAT; then fn_echo " - tomcat $TOMCAT_VERSION"; fi if $INSTALL_TOMCAT; then fn_echo " - tomcat $TOMCAT_VERSION"; fi
if $INSTALL_GEOSERVER; then fn_echo " - geoserver $GEOSERVER_VERSION"; fi if $INSTALL_GEOSERVER; then fn_echo " - geoserver $GEOSERVER_VERSION"; fi
if $INSTALL_POSTGRES_ADMIN; then fn_echo " - phpPgAdmin"; fi if $INSTALL_PGADMIN; then fn_echo " - pgadmin4"; fi
if $INSTALL_OSMCONVERT; then fn_echo " - osmconvert"; fi if $INSTALL_OSMCONVERT; then fn_echo " - osmconvert"; fi
if $INSTALL_OSMOSIS; then fn_echo " - osmosis $OSMOSIS_VERSION"; fi if $INSTALL_OSMOSIS; then fn_echo " - osmosis $OSMOSIS_VERSION"; fi
if $IMPORT_DATA_NEO4J_SPATIAL; then fn_echo "Importing data from OSM"; fi if $IMPORT_DATA_NEO4J_SPATIAL; then fn_echo "Importing data from OSM"; fi
...@@ -424,9 +424,15 @@ if $INSTALL_POSTGIS; then ...@@ -424,9 +424,15 @@ if $INSTALL_POSTGIS; then
fi fi
fi fi
if $INSTALL_POSTGRES_ADMIN; then if $INSTALL_PGADMIN; then
fn_echo "Installing phpPgAdmin" # Server mode installation disabled by now (does not work right now... might need python3.5)
yum -y install phpPgAdmin >> $WORKING_DIR/install_phpPgAdmin.log 2>&1 fn_echo "Installing pgadmin4"
yum -y install pgadmin4 >> $WORKING_DIR/install_pgadmin.log 2>&1
#yum -y install mod_wsgi
#sed -i "s/SERVER_MODE = True/SERVER_MODE = False/" /usr/lib/*/site-packages/pgadmin4-web/config.py
sed -i "s/DEFAULT_SERVER = 'localhost'/DEFAULT_SERVER = '0.0.0.0'/" /usr/lib/*/site-packages/pgadmin4-web/config.py
mv /etc/httpd/conf.d/pgadmin4.conf /etc/httpd/conf.d/pgadmin4.conf.unused
fi fi
if $INSTALL_JAVA; then if $INSTALL_JAVA; then
...@@ -718,11 +724,11 @@ if $INSTALL_POSTGRES; then ...@@ -718,11 +724,11 @@ if $INSTALL_POSTGRES; then
cp "$SHARED_CONF_DIR/postgresql.conf" "$POSTGRESQL_GLOB_CONF" cp "$SHARED_CONF_DIR/postgresql.conf" "$POSTGRESQL_GLOB_CONF"
fi fi
if $INSTALL_POSTGRES_ADMIN; then #if $INSTALL_PGADMIN; then
fn_echo "Configuring phpPgAdmin" #fn_echo "Configuring pgadmin"
cp $SHARED_CONF_DIR/phpPgAdmin.conf /etc/httpd/conf.d/phpPgAdmin.conf #cp $SHARED_CONF_DIR/phpPgAdmin.conf /etc/httpd/conf.d/phpPgAdmin.conf
sed -i 's/\$conf\['\''servers'\''\]\[0\]\['\''host'\''\] = '\'''\''/\$conf\['\''servers'\''\]\[0\]\['\''host'\''\] = '\''localhost'\''/' /usr/share/phpPgAdmin/conf/config.inc.php #sed -i 's/\$conf\['\''servers'\''\]\[0\]\['\''host'\''\] = '\'''\''/\$conf\['\''servers'\''\]\[0\]\['\''host'\''\] = '\''localhost'\''/' /usr/share/phpPgAdmin/conf/config.inc.php
fi #fi
if $INSTALL_NEO4J; then if $INSTALL_NEO4J; then
fn_echo "Initializing neo4j database" fn_echo "Initializing neo4j database"
...@@ -810,6 +816,9 @@ fi ...@@ -810,6 +816,9 @@ fi
if $INSTALL_POSTGRES; then if $INSTALL_POSTGRES; then
fn_service_start "$POSTGRESQL_SERVICE_NAME" fn_service_start "$POSTGRESQL_SERVICE_NAME"
fi fi
if $INSTALL_PGADMIN; then
fn_service_start "pgadmin4"
fi
if $INSTALL_NEO4J; then if $INSTALL_NEO4J; then
fn_service_start "$NEO4J_SERVICE_NAME" fn_service_start "$NEO4J_SERVICE_NAME"
fi fi
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment