Skip to content
Snippets Groups Projects
Commit 1eed77be authored by User expired's avatar User expired
Browse files

improved error handling in bash script

parent 6cd11f37
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
CHANGELOG.md
+ 1
− 0
View file @ 1eed77be
Upcoming version:
-----------------
- improved error handling in bash script (Nikolaus Krismer)
- improved some messages and the documentation (Nikolaus Krismer)
- added script createDataset (in folder etc) to allow creation of datasets on local machine (Nikolaus Krismer)
- re-added some parts for neo4j-spatial and geoserver (Nikolaus Krismer)
......
bootstrap/importData.sh
+ 80
− 46
View file @ 1eed77be
......@@ -34,7 +34,7 @@ fi
fn_arg2string PG_DB_USER "${1}" "${PG_DB_USER}"
fn_arg2string PG_DB_PASSWORD "${2}" "${PG_DB_PASSWORD}"
fn_arg2boolean DEPLOY_ALL_DATASETS "${3}" "${DEPLOY_ALL_DATASETS}"
fn_arg2boolean DEPLOY_ANY_DATASETS "${4}" "true"
fn_arg2boolean DEPLOY_ANY_DATASET "${4}" "true"
######################################
# Configuration (tool versions, ...) #
......@@ -54,6 +54,36 @@ OSM_KEYS="aerialway,highway,public_transport,railway,route"
# Function definitions #
############################
function fn_create_synthetic_networks() {
if $AVAILABLE_ISOCHRONE_TOOLS; then
echo "Importing data for synthetic networks"
# GridNetwork
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the grid network. The process will exit now with status $?"
exit $?
fi
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d "$DENSITY" >> "$WORKING_DIR/generate_gridNetwork.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the node_density table for 'grid_s100' using isochrone-tools. The process will exit now with status $?"
exit $?
fi
# SpiderNetwork
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the spider network. The process will exit now with status $?"
exit $?
fi
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d "$DENSITY" >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the node_density table for 'spirder_l1000' using isochrone-tools. The process will exit now with status $?"
exit $?
fi
fi
}
function fn_import_dataset() {
local NAME="$1"
local SRID="$2"
......@@ -78,6 +108,10 @@ function fn_import_dataset_neo4j_spatial() {
# We copy data from postgis for specified dataset (after postgis data import)
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.exporter.ImportDataNeo4j -t "${CITY}" > "$WORKING_DIR/import_neo4j_$CITY.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while importing data into neo4j using isochrone-tools. The process will exit now with status $?"
exit $?
fi
}
function fn_import_dataset_postgis() {
......@@ -114,6 +148,10 @@ function fn_import_dataset_postgis() {
echo " - creating datamodel using isochrone-datamodel"
# Create datamodel using isochrone-datamodel project
DB_USERNAME="$PG_DB_USER" DB_PASSWORD="$PG_DB_PASSWORD" "$DEPLOY_DIR/isochrone-datamodel/builder.sh" -d -s -b -l -t${SRID} -c${CITY} >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating datamodel using isochrone-datamodel. The process will exit now with status $?"
exit $?
fi
echo " - copying tables to isochrone database"
# If working with multiple TARGET_SRIDs we have to delete an eventually existing nodes_density table here, so on import later a DROP TABLE ${CITY}_nodes works wihtout CASCADE (pg_dump does not use CASCADE)
......@@ -123,6 +161,10 @@ function fn_import_dataset_postgis() {
if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then
java -cp "${DEPLOY_DIR}/isochrone-tools.jar" "at.uibk.dbis.isochrone.generator.density.DensityGenerator" -t "${CITY}" -d "${DENSITY}" >> "$WORKING_DIR/create_datamodel_${CITY}_${SRID}.log" 2>&1
if [ $? -ne 0 ]; then
printf "%s\n" "[ERROR] An error occurred while creating the node_density table for '${CITY}' using isochrone-tools. The process will exit now with status $?"
exit $?
fi
fi
if $AVAILABLE_ISOCHRONE_DATAMODEL; then
......@@ -256,57 +298,49 @@ function fn_filter_osm_data() {
# Data import #
################
if ! $DEPLOY_ANY_DATASETS; then
return;
fi
if $DEPLOY_ANY_DATASET; then
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Importing datasets ($(date +%H:%M:%S)):"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Importing datasets ($(date +%H:%M:%S)):"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
START=$(date +%s)
START=$(date +%s)
mkdir -p $DOWNLOAD_DIR
mkdir -p $WORKING_DIR
cd "$WORKING_DIR"
mkdir -p $DOWNLOAD_DIR
mkdir -p $WORKING_DIR
cd "$WORKING_DIR"
# Generate synthetic datasets (and densities for them) -> they are always created from scratch (never cached)
fn_create_synthetic_networks
# Generate synthetic datasets (and densities for them) -> they are always created from scratch (never cached)
if $AVAILABLE_POSTGIS && $AVAILABLE_ISOCHRONE_TOOLS; then
echo "Importing data for synthetic networks"
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.GridNetworkGenerator -d 100 -l 60 > "$WORKING_DIR/generate_gridNetwork.log" 2>&1
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.SpiderNetworkGenerator -d 6 -lvl 1000 -l 60 > "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t grid_s100 -d "$DENSITY" >> "$WORKING_DIR/generate_gridNetwork.log" 2>&1
java -cp ${DEPLOY_DIR}/isochrone-tools.jar at.uibk.dbis.isochrone.generator.density.DensityGenerator -t spider_l1000 -d "$DENSITY" >> "$WORKING_DIR/generate_spiderNetwork.log" 2>&1
fi
SRID_ARR=(${TARGET_SRID//,/ })
for CURRENT_SRID in "${SRID_ARR[@]}"; do
# Import real world datasets
fn_import_dataset "Bolzano" "$CURRENT_SRID"
fn_import_dataset "Innsbruck" "$CURRENT_SRID"
fn_import_dataset "Salzburg" "$CURRENT_SRID"
fn_import_dataset "SanFrancisco" "$CURRENT_SRID"
fn_import_dataset "WashingtonDC" "$CURRENT_SRID"
if $DEPLOY_ALL_DATASETS; then
fn_import_dataset "Italy" "$CURRENT_SRID"
fn_import_dataset "AltoAdige" "$CURRENT_SRID"
fi
SRID_ARR=(${TARGET_SRID//,/ })
for CURRENT_SRID in "${SRID_ARR[@]}"; do
# Import real world datasets
fn_import_dataset "Bolzano" "$CURRENT_SRID"
fn_import_dataset "Innsbruck" "$CURRENT_SRID"
fn_import_dataset "Salzburg" "$CURRENT_SRID"
fn_import_dataset "SanFrancisco" "$CURRENT_SRID"
fn_import_dataset "WashingtonDC" "$CURRENT_SRID"
if $DEPLOY_ALL_DATASETS; then
fn_import_dataset "Italy" "$CURRENT_SRID"
fn_import_dataset "AltoAdige" "$CURRENT_SRID"
fi
# Not importing datasets one-by-one into spatialite -> we copy data from postgis (after postgis data import)
if $IMPORT_DATA_SPATIALITE; then
fn_import_spatialite "$CURRENT_SRID"
fi
done
# Not importing datasets one-by-one into spatialite -> we copy data from postgis (after postgis data import)
if $IMPORT_DATA_SPATIALITE; then
fn_import_spatialite "$CURRENT_SRID"
# After data import we have to initialize geoserver layers (if geoserver is used)
if $AVAILABLE_GEOSERVER; then
fn_init_geoserver
fi
done
# After data import we have to initialize geoserver layers (if geoserver is used)
if $AVAILABLE_GEOSERVER; then
fn_init_geoserver
fi
END=$(date +%s)
TOTAL=$(( $END - $START ))
END=$(date +%s)
TOTAL=$(( $END - $START ))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Datasets imported"
printf ' Time to import the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo " Datasets imported"
printf ' Time to import the datasets: %dh:%dm:%ds\n' $(($TOTAL/3600)) $(($TOTAL%3600/60)) $(($TOTAL%60))
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
fi
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment